Revamp spike PCA figure panels and stats

.codeocean/datasets.json

@@ -4,6 +4,10 @@
 		{
 			"id": "68ef27d7-9d95-40ce-9e40-7de93dccf5f8",
 			"mount": "LCNE-patchseq-ephys"
+		},
+		{
+			"id": "c5d5e922-1863-4869-99ff-fdce3e10edc7",
+			"mount": "fonts"
 		}
 	]
 }

code/run

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -ex
+
+# This is the master script for the capsule. When you click "Reproducible Run", the code in this file will execute.
+python -m LCNE_patchseq_analysis.figures.main_pca_tau

environment/Dockerfile

@@ -15,8 +15,8 @@ ARG GIT_ASKPASS
 ARG GIT_ACCESS_TOKEN
 COPY git-askpass /
 
-RUN pip install --no-cache-dir "git+https://github.com/AllenNeuralDynamics/LCNE-patchseq-analysis.git@main#egg=LCNE-patchseq-analysis[panel]"
-
+RUN pip install -U --no-cache-dir \
+    "git+https://github.com/AllenNeuralDynamics/LCNE-patchseq-analysis.git@b8c709294e970e35beda633757255b00d579f832#egg=LCNE-patchseq-analysis"
 
 COPY postInstall /
 RUN /postInstall

environment/postInstall

@@ -1,29 +1,17 @@
 #!/usr/bin/env bash
 set -e
 
-# check if code-server is installed, and then install extensions into specified directory
-if code-server --disable-telemetry --version; then
-			 if [ ! -d "/.vscode/extensions" ]
-				then
-				   echo "Directory /.vscode/extensions DOES NOT exists."
-				   mkdir -p /.vscode/extensions/
-			 fi
-
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension ms-python.python
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension ms-toolsai.jupyter
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension njpwerner.autodocstring
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension KevinRose.vsc-python-indent
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension mhutchie.git-graph
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension zhuangtongfa.material-theme
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension ms-python.black-formatter
-	  	 code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension randomfractalsinc.vscode-data-preview
-	  	 # code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension ryanluker.vscode-coverage-gutters
-
-       curl -L -o copilot_1.161.zip https://github.com/user-attachments/files/16859733/copilot_1.161.zip
-       unzip copilot_1.161.zip
-       code-server --disable-telemetry --extensions-dir=/.vscode/extensions --install-extension GitHub.copilot-1.161.0.vsix
-       rm copilot_1.161.zip GitHub.copilot-1.161.0.vsix
-
-       else
-          echo "code-server not found"
-       fi
+HOME_DIR="${HOME:-/root}"
+BASHRC="${HOME_DIR}/.bashrc"
+
+cat >> "$BASHRC" <<'EOF'
+
+# LCNE editable install (runs once on first login, src/ only available at runtime)
+if [ ! -f "$HOME/.lcne_editable_setup_done" ]; then
+    echo "Installing local package in editable mode from /root/capsule..."
+    pip install -e /root/capsule
+    echo "Editable install complete"
+
+    touch "$HOME/.lcne_editable_setup_done"
+fi
+EOF

pyproject.toml

@@ -27,7 +27,9 @@ dependencies = [
     'requests',
     'trimesh',
     'statsmodels',
-    'matplotlib_venn'
+    'matplotlib_venn',
+    'scipy',
+    'scikit-learn',
 ]
 
 [project.optional-dependencies]

src/LCNE_patchseq_analysis/efel/core.py

@@ -159,9 +159,7 @@ def extract_spike_waveforms(
         DataFrame containing spike waveforms
     """
     peak_times = (
-        features_dict["df_features_per_spike"]
-        .reset_index()
-        .set_index("sweep_number")["peak_time"]
+        features_dict["df_features_per_spike"].reset_index().set_index("sweep_number")["peak_time"]
     )
 
     # Time can be determined by the sampling rate
@@ -176,9 +174,9 @@ def extract_spike_waveforms(
         v = raw_trace["V"]
 
         for peak_time in peak_times_this_sweep:
-            idx = np.where(
-                (t >= peak_time + spike_window[0]) & (t < peak_time + spike_window[1])
-            )[0]
+            idx = np.where((t >= peak_time + spike_window[0]) & (t < peak_time + spike_window[1]))[
+                0
+            ]
             v_this = v[idx]
             vs.append(v_this)
 
@@ -312,9 +310,7 @@ def extract_features_using_efel(
     # Append stimulus to raw_traces (doing here because eFEL cannot handle it)
     for raw_trace in raw_traces:
         raw_trace["stimulus"] = raw.get_stimulus(raw_trace["sweep_number"][0])
-    df_peri_stimulus_raw_traces = extract_peri_stimulus_raw_traces(
-        raw_traces, features_dict
-    )
+    df_peri_stimulus_raw_traces = extract_peri_stimulus_raw_traces(raw_traces, features_dict)
 
     # -- Enrich df_sweeps --
     df_sweeps = raw.df_sweeps.copy()
@@ -323,9 +319,9 @@ def extract_features_using_efel(
         "spike_count": "efel_num_spikes",
         "first_spike_AP_width": "efel_first_spike_AP_width",
     }
-    _df_to_df_sweeps = features_dict["df_features_per_sweep"][
-        list(col_to_df_sweeps.keys())
-    ].rename(columns=col_to_df_sweeps)
+    _df_to_df_sweeps = features_dict["df_features_per_sweep"][list(col_to_df_sweeps.keys())].rename(
+        columns=col_to_df_sweeps
+    )
     df_sweeps = df_sweeps.merge(_df_to_df_sweeps, on="sweep_number", how="left")
 
     # Add metadata to features_dict
@@ -377,9 +373,7 @@ def extract_efel_one(
     except Exception as e:
         import traceback
 
-        error_message = (
-            f"Error processing {ephys_roi_id}: {str(e)}\n{traceback.format_exc()}"
-        )
+        error_message = f"Error processing {ephys_roi_id}: {str(e)}\n{traceback.format_exc()}"
         logger.error(error_message)
         return error_message
 

src/LCNE_patchseq_analysis/efel/pipeline.py

@@ -15,19 +15,15 @@
 logger = logging.getLogger(__name__)
 
 
-def extract_efel_features_in_parallel(
-    skip_existing: bool = True, skip_errors: bool = True
-):
+def extract_efel_features_in_parallel(skip_existing: bool = True, skip_errors: bool = True):
     """Extract eFEL features in parallel."""
 
     def get_roi_ids():
         df_meta = load_ephys_metadata(if_from_s3=False, combine_roi_ids=True)
         return df_meta["ephys_roi_id_tab_master"]
 
     def check_existing(ephys_roi_id):
-        return os.path.exists(
-            f"{RESULTS_DIRECTORY}/features/{int(ephys_roi_id)}_efel.h5"
-        )
+        return os.path.exists(f"{RESULTS_DIRECTORY}/features/{int(ephys_roi_id)}_efel.h5")
 
     return run_parallel_processing(
         process_func=extract_efel_one,
@@ -39,15 +35,11 @@ def check_existing(ephys_roi_id):
     )
 
 
-def generate_sweep_plots_in_parallel(
-    skip_existing: bool = True, skip_errors: bool = True
-):
+def generate_sweep_plots_in_parallel(skip_existing: bool = True, skip_errors: bool = True):
     """Generate sweep plots in parallel."""
 
     def check_existing(ephys_roi_id):
-        return os.path.exists(
-            f"{RESULTS_DIRECTORY}/plots/{int(ephys_roi_id)}/all_success"
-        )
+        return os.path.exists(f"{RESULTS_DIRECTORY}/plots/{int(ephys_roi_id)}/all_success")
 
     return run_parallel_processing(
         process_func=generate_sweep_plots_one,
@@ -58,9 +50,7 @@ def check_existing(ephys_roi_id):
     )
 
 
-def extract_cell_level_stats_in_parallel(
-    skip_errors: bool = True, if_generate_plots: bool = True
-):
+def extract_cell_level_stats_in_parallel(skip_errors: bool = True, if_generate_plots: bool = True):
     """Extract cell-level statistics from all available eFEL features files in parallel."""
 
     # ---- Extract cell-level stats ----
@@ -111,9 +101,9 @@ def extract_cell_level_stats_in_parallel(
     )
 
     # ---- Merge into Brian's spreadsheet ----
-    df_ephys_metadata = load_ephys_metadata(
-        if_from_s3=False, combine_roi_ids=True
-    ).rename(columns={"ephys_roi_id_tab_master": "ephys_roi_id"})
+    df_ephys_metadata = load_ephys_metadata(if_from_s3=False, combine_roi_ids=True).rename(
+        columns={"ephys_roi_id_tab_master": "ephys_roi_id"}
+    )
     df_merged = df_ephys_metadata.merge(df_cell_stats, on="ephys_roi_id", how="left")
 
     # ---- Post-processing ----
@@ -127,9 +117,7 @@ def extract_cell_level_stats_in_parallel(
     # Remove "first_spike_" in all column names
     df_merged.columns = [col.replace("first_spike_", "") for col in df_merged.columns]
     # Add EFEL_prefix to all columns that has @ in its name
-    df_merged.columns = [
-        f"efel_{col}" if "@" in col else col for col in df_merged.columns
-    ]
+    df_merged.columns = [f"efel_{col}" if "@" in col else col for col in df_merged.columns]
 
     # ---- Save the summary table to disk ----
     save_path = f"{RESULTS_DIRECTORY}/cell_stats/cell_level_stats.csv"
@@ -148,9 +136,7 @@ def extract_cell_level_stats_in_parallel(
     save_path = f"{RESULTS_DIRECTORY}/cell_stats/cell_level_last_spike_waveforms.pkl"
     df_cell_representative_last_spike_waveforms.to_pickle(save_path)
 
-    logger.info(
-        f"Successfully extracted cell-level stats for {len(df_cell_stats)} cells!"
-    )
+    logger.info(f"Successfully extracted cell-level stats for {len(df_cell_stats)} cells!")
     logger.info(f"Summary table saved to {save_path}")
 
     return df_merged