Segfault when trying to use the yfinance package.

[jjbrosnan]

Description

Attempting to pull market data into DHC with yfinance results in a segfault.

Steps to reproduce

Run DHC v0.37.4 via Docker, pip install both yfinance and psutil, then run this script:

import logging
import sys
import traceback
import psutil
import threading
import queue
import time

import yfinance as yf
import pandas as pd
import numpy as np
from datetime import datetime

from deephaven import ui, agg, empty_table, updateby as uby
from deephaven.stream.table_publisher import table_publisher
from deephaven import dtypes as dht
from deephaven.plot import express as dx
from deephaven import pandas as dhpd

# Logging Configuration
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('/var/log/stock_dashboard.log'),
        logging.StreamHandler(sys.stdout)
    ]
)

def log_system_resources():
    while True:
        try:
            memory = psutil.virtual_memory()
            cpu_percent = psutil.cpu_percent()
            
            logging.info(f"System Resources: "
                         f"CPU: {cpu_percent}% | "
                         f"Memory: {memory.percent}% used "
                         f"({memory.used/1024/1024:.2f} MB)")
        
        except Exception as e:
            logging.error(f"Resource logging error: {e}")
        
        time.sleep(60)

def initialize_stock_data(symbols=['NVDA'], period='1d', interval='1m'):
    try:
        all_data = []
        for symbol in symbols:
            try:
                ticker = yf.Ticker(symbol)
                hist = ticker.history(period=period, interval=interval)
                if not hist.empty:
                    hist['Sym'] = symbol
                    hist['Exchange'] = 'NASDAQ'
                    hist = hist.reset_index()
                    hist = hist[['Datetime', 'Close', 'Volume', 'Sym', 'Exchange']]
                    hist = hist.rename(columns={'Datetime': 'Timestamp', 'Close': 'Price'})
                    all_data.append(hist)
                else:
                    logging.warning(f"No historical data for {symbol}")
            except Exception as e:
                logging.error(f"Error fetching data for {symbol}: {e}")
        
        if not all_data:
            logging.critical("No stock data could be fetched!")
            return empty_table(1).update([
                "Timestamp=null", "Price=0.0", "Volume=0", "Sym=``", "Exchange=``"
            ])
        
        combined_data = pd.concat(all_data)
        return dhpd.to_table(combined_data)
    
    except Exception as e:
        logging.critical(f"Catastrophic error in stock data initialization: {e}")
        traceback.print_exc()
        return empty_table(1)

def create_stock_publisher():
    return table_publisher(
        "Stock Data",
        {
            "Timestamp": dht.datetime,
            "Price": dht.double,
            "Volume": dht.long,
            "Sym": dht.string,
            "Exchange": dht.string
        }
    )

@ui.component
def options_panel(symbol="NVDA"):
    try:
        ticker = yf.Ticker(symbol)
        options = ticker.options
        
        if not options:
            logging.warning(f"No options data available for {symbol}")
            return ui.text("No options data available")
        
        expiry = options[0]
        chain = ticker.option_chain(expiry)
        
        calls_data = chain.calls[['strike', 'lastPrice', 'impliedVolatility', 'openInterest']]
        calls_table = dhpd.to_table(calls_data)
        
        iv_plot = dx.scatter(
            calls_table, 
            x="strike", 
            y="impliedVolatility", 
            size="openInterest",
            title="Options Implied Volatility"
        )
        
        return ui.panel(
            ui.stack(
                ui.panel(calls_table, title="Options Chain"),
                ui.panel(iv_plot, title="IV Analysis")
            ),
            title=f"{symbol} Options"
        )
    except Exception as e:
        logging.error(f"Error loading options data for {symbol}: {e}")
        return ui.text(f"Error loading options data: {e}")

def compute_rolling_stats(source_table):
    try:
        return source_table \
            .update_by([
                uby.rolling_avg_time("Timestamp", "PriceAvg5s=Price", "PT2.5s", "PT2.5s"),
                uby.rolling_avg_time("Timestamp", "PriceAvg30s=Price", "PT15s", "PT15s"),
                uby.rolling_std_time("Timestamp", "PriceStd5s=Price", "PT2.5s", "PT2.5s"),
                uby.rolling_std_time("Timestamp", "PriceStd30s=Price", "PT15s", "PT15s"),
            ], by=["Sym"])
    except Exception as e:
        logging.error(f"Error computing rolling statistics: {e}")
        return source_table

@ui.component
def line_plot(filtered_source, exchange, window_size, bol_bands):
    try:
        window_size_key = {
            "5 seconds": ("PriceAvg5s", "PriceStd5s"),
            "30 seconds": ("PriceAvg30s", "PriceStd30s"),
        }
        
        bol_bands_key = {"None": None, "80%": 1.282, "90%": 1.645, "95%": 1.960, "99%": 2.576}
        
        base_plot = ui.use_memo(
            lambda: dx.line(
                filtered_source, 
                x="Timestamp", 
                y="Price", 
                by="Exchange" if exchange == "All" else None,
                unsafe_update_figure=lambda fig: fig.update_traces(opacity=0.4)
            ), 
            [filtered_source, exchange]
        )
        
        window_size_avg_key_col = window_size_key.get(window_size, ("PriceAvg30s", "PriceStd30s"))[0]
        window_size_std_key_col = window_size_key.get(window_size, ("PriceAvg30s", "PriceStd30s"))[1]
        
        avg_plot = ui.use_memo(
            lambda: dx.line(
                filtered_source,
                x="Timestamp", 
                y=window_size_avg_key_col,
                color_discrete_sequence=["orange"],
                labels={window_size_avg_key_col: "Rolling Average"}
            ),
            [filtered_source, window_size_avg_key_col]
        )
        
        bol_bands_key_col = bol_bands_key.get(bol_bands)
        
        bol_plot = ui.use_memo(
            lambda: dx.line(
                filtered_source.update([
                    f"errorY={window_size_avg_key_col} + {bol_bands_key_col}*{window_size_std_key_col}",
                    f"errorYMinus={window_size_avg_key_col} - {bol_bands_key_col}*{window_size_std_key_col}",
                ]),
                x="Timestamp", 
                y=["errorYMinus", "errorY"],
                color_discrete_sequence=["rgba(255,165,0,0.3)", "rgba(255,165,0,0.3)"]
            ) if bol_bands_key_col is not None else None,
            [filtered_source, window_size_avg_key_col, window_size_std_key_col, bol_bands_key_col]
        )
        
        plot = ui.use_memo(
            lambda: dx.layer(base_plot, avg_plot, bol_plot), 
            [base_plot, avg_plot, bol_plot]
        )
        
        return ui.panel(plot, title="Prices")
    except Exception as e:
        logging.error(f"Error creating line plot: {e}")
        return ui.text(f"Error creating plot: {e}")

# Remaining components and dashboard layout remain the same as in original script

# Initialization with logging
stocks = initialize_stock_data()
stock_publisher, publisher = create_stock_publisher()

# Start resource monitoring
threading.Thread(target=log_system_resources, daemon=True).start()

# Dashboard creation remains the same
dashboard = ui.dashboard(financial_dashboard_layout(stocks, _stocks_with_stats))

Expected results

The container to not exit with a segfault. The code may or may not work as-is, but it can't be verified.

Actual results

A segfault, causing the Docker container to exit immediately.

Additional details and attachments

Docker logs:

# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007fe739c4be80, pid=1, tid=1072
#
# JRE version: OpenJDK Runtime Environment Temurin-21.0.5+11 (21.0.5+11) (build 21.0.5+11-LTS)
# Java VM: OpenJDK 64-Bit Server VM Temurin-21.0.5+11 (21.0.5+11-LTS, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)
# Problematic frame:
# C  [libpython3.10.so.1.0+0x222e80]  PyObject_GC_Del+0x40
#
# Core dump will be written. Default location: Core dumps may be processed with "/usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h" (or dumping to //core.1)
#
# An error report file with more information is saved as:
# //hs_err_pid1.log
[583.212s][warning][os] Loading hsdis library failed
#
# If you would like to submit a bug report, please visit:
#   https://github.com/adoptium/adoptium-support/issues
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.

Core dump:

root@18f76a0fd819:/# grep -i -A 50 -B 50 "core dump" //hs_err_pid1.log
#
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007fe01c7f6e80, pid=1, tid=2141
#
# JRE version: OpenJDK Runtime Environment Temurin-21.0.5+11 (21.0.5+11) (build 21.0.5+11-LTS)
# Java VM: OpenJDK 64-Bit Server VM Temurin-21.0.5+11 (21.0.5+11-LTS, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)
# Problematic frame:
# C  [libpython3.10.so.1.0+0x222e80]  PyObject_GC_Del+0x40
#
# Core dump will be written. Default location: Core dumps may be processed with "/usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h" (or dumping to //core.1)
#
# If you would like to submit a bug report, please visit:
#   https://github.com/adoptium/adoptium-support/issues
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#

---------------  S U M M A R Y ------------

Command Line: --add-exports=java.management/sun.management=ALL-UNNAMED --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED -Dauthentication.client.configuration.list=AuthHandlers --add-opens=java.base/java.nio=ALL-UNNAMED -XX:+UnlockDiagnosticVMOptions -XX:CompilerDirectivesFile=/opt/deephaven/server-jetty-0.37.4/etc/dh-compiler-directives.txt -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:+UseStringDeduplication -XX:GCLockerRetryAllocationCount=128 -Xmx8g -Dauthentication.psk=Password123 io.deephaven.server.jetty.JettyMain

Host: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz, 1 cores, 949M, Ubuntu 22.04.5 LTS
Time: Fri Jan 31 15:44:13 2025 UTC elapsed time: 522.778403 seconds (0d 0h 8m 42s)

---------------  T H R E A D  ---------------

Current thread (0x00007fe041625680):  JavaThread "DeephavenApiServer-Scheduler-Serial-1" daemon [_thread_in_native, id=2141, stack(0x00007fdfa15fe000,0x00007fdfa16fe000) (1024K)]

Stack: [0x00007fdfa15fe000,0x00007fdfa16fe000],  sp=0x00007fdfa16fb8c8,  free space=1014k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C  [libpython3.10.so.1.0+0x222e80]  PyObject_GC_Del+0x40
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j  org.jpy.PyLib.executeCode(Ljava/lang/String;ILjava/lang/Object;Ljava/lang/Object;)J+0
j  org.jpy.PyObject.executeCode(Ljava/lang/String;Lorg/jpy/PyInputMode;Ljava/lang/Object;Ljava/lang/Object;)Lorg/jpy/PyObject;+25
j  io.deephaven.engine.util.PythonEvaluatorJpy.evalScript(Ljava/lang/String;)V+14
j  io.deephaven.integrations.python.PythonDeephavenSession.lambda$evaluate$1(Ljava/lang/String;)V+5
j  io.deephaven.integrations.python.PythonDeephavenSession$$Lambda+0x00007fdfc8654fc0.run()V+8
j  io.deephaven.util.locks.FunctionalLock.doLockedInterruptibly(Lio/deephaven/util/function/ThrowingRunnable;)V+7
j  io.deephaven.integrations.python.PythonDeephavenSession.evaluate(Ljava/lang/String;Ljava/lang/String;)V+42
j  io.deephaven.engine.util.AbstractScriptSession.lambda$evaluateScript$0(Ljava/lang/String;Ljava/lang/String;)V+3
j  io.deephaven.engine.util.AbstractScriptSession$$Lambda+0x00007fdfc8654da8.run()V+12
j  io.deephaven.engine.context.ExecutionContext.lambda$apply$0(Ljava/lang/Runnable;)Ljava/lang/Object;+1
j  io.deephaven.engine.context.ExecutionContext$$Lambda+0x00007fdfc8153030.get()Ljava/lang/Object;+4
j  io.deephaven.engine.context.ExecutionContext.apply(Ljava/util/function/Supplier;)Ljava/lang/Object;+6
j  io.deephaven.engine.context.ExecutionContext.apply(Ljava/lang/Runnable;)V+7
j  io.deephaven.engine.util.AbstractScriptSession.evaluateScript(Ljava/lang/String;Ljava/lang/String;)Lio/deephaven/engine/util/ScriptSession$Changes;+50
j  io.deephaven.engine.util.DelegatingScriptSession.evaluateScript(Ljava/lang/String;Ljava/lang/String;)Lio/deephaven/engine/util/ScriptSession$Changes;+7
j  io.deephaven.engine.util.ScriptSession.evaluateScript(Ljava/lang/String;)Lio/deephaven/engine/util/ScriptSession$Changes;+3
j  io.deephaven.server.console.ConsoleServiceGrpcImpl.lambda$executeCommand$7(Lio/deephaven/server/session/SessionState$ExportObject;Lio/deephaven/proto/backplane/script/grpc/ExecuteCommandRequest;)Lio/deephaven/proto/backplane/script/grpc/ExecuteCommandResponse;+65
j  io.deephaven.server.console.ConsoleServiceGrpcImpl$$Lambda+0x00007fdfc867a678.call()Ljava/lang/Object;+8
j  io.deephaven.server.session.SessionState$ExportObject.doExport()V+225
j  io.deephaven.server.session.SessionState$ExportObject$$Lambda+0x00007fdfc867a890.run()V+4
J 3399 c1 java.util.concurrent.Executors$RunnableAdapter.call()Ljava/lang/Object; [email protected] (14 bytes) @ 0x00007fe028dda2dc [0x00007fe028dda1c0+0x000000000000011c]
J 3627 c1 java.util.concurrent.FutureTask.run()V [email protected] (123 bytes) @ 0x00007fe028e3789c [0x00007fe028e37140+0x000000000000075c]
j  java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+92 [email protected]
j  java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5 [email protected]
j  io.deephaven.server.runner.scheduler.SchedulerModule$ThreadFactory.lambda$newThread$0(Ljava/lang/Runnable;)V+18
j  io.deephaven.server.runner.scheduler.SchedulerModule$ThreadFactory$$Lambda+0x00007fdfc84ebd28.run()V+8
j  java.lang.Thread.runWith(Ljava/lang/Object;Ljava/lang/Runnable;)V+5 [email protected]
j  java.lang.Thread.run()V+19 [email protected]

Versions

• Deephaven: 0.37.4 (latest)
• OS: OS X (originally discovered on AWS EC2 - not sure what OS on the instance)
• Browser: Chrome/unsure
• Docker: 20.10.13/unsure