Skip to content

Commit 929abc5

Browse files
kjsandbrinkMMathisLab
kjsandbrink
authored and
MMathisLab
committed
code cleanup for publication
1 parent 3230b3d commit 929abc5

23 files changed

Lines changed: 775 additions & 1378 deletions

‎single_cell/01 Save Hidden Activations.ipynb‎

Lines changed: 119 additions & 122 deletions
Large diffs are not rendered by default.

‎single_cell/02 CKA.ipynb‎

Lines changed: 139 additions & 139 deletions
Large diffs are not rendered by default.

‎single_cell/03 tSNE.ipynb‎

Lines changed: 74 additions & 60 deletions
Large diffs are not rendered by default.

‎single_cell/04 Demo Tuning Curve Fit.ipynb‎

Lines changed: 99 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,18 @@
1111
},
1212
{
1313
"cell_type": "code",
14-
"execution_count": 11,
14+
"execution_count": 1,
1515
"metadata": {},
16-
"outputs": [],
16+
"outputs": [
17+
{
18+
"name": "stdout",
19+
"output_type": "stream",
20+
"text": [
21+
"No module named 'tensorflow.python.tools'\n",
22+
"proceeding without savelouts , this will only work if no data is being generated\n"
23+
]
24+
}
25+
],
1726
"source": [
1827
"from sklearn.metrics import r2_score, mean_squared_error\n",
1928
"from sklearn.model_selection import train_test_split\n",
@@ -23,12 +32,14 @@
2332
"import pickle, os, argparse\n",
2433
"import multiprocessing as mp\n",
2534
"import h5py\n",
35+
"import matplotlib\n",
2636
"from sklearn.metrics import roc_auc_score\n",
2737
"from sklearn.preprocessing import label_binarize\n",
2838
"from sklearn.svm import LinearSVC\n",
2939
"from sklearn.multiclass import OneVsRestClassifier\n",
30-
"from controls_main import RunInfo\n",
31-
"from rowwise_neuron_curves_controls import compute_metrics, lstring, get_binidx, get_centers, unit_vector, \\\n",
40+
"from rowwise_neuron_curves import lstring, read_layer_reps\n",
41+
"from main import RunInfo\n",
42+
"from rowwise_neuron_curves import compute_metrics, lstring, get_binidx, get_centers, unit_vector, \\\n",
3243
" angle_xaxis, get_polar, linreg, feature_set, X_data\n",
3344
"\n",
3445
"# GLOBAL PARS\n",
@@ -47,33 +58,45 @@
4758
},
4859
{
4960
"cell_type": "code",
50-
"execution_count": 12,
61+
"execution_count": 13,
5162
"metadata": {},
5263
"outputs": [],
5364
"source": [
5465
"modelinfo = dict({'type': 'S',\n",
55-
" 'base': 'spatial_temporal_4_8-16-16-32_64-64-64-64_5272',\n",
56-
" 'name': 'spatial_temporal_4_8-16-16-32_64-64-64-64_5272_1',\n",
57-
" 'shortbase': 'Spatial-Temporal',\n",
58-
" 'shortname': 'Spatial-Temporal_1',\n",
59-
" 'nlayers': 8,\n",
60-
" 'max_act': 14,\n",
61-
" 'control': False,\n",
62-
" 'cmap': 'Blues_r',\n",
63-
" 'color': 'C0',\n",
64-
" 'control_cmap': 'Purples_r'})\n",
66+
" 'typename': 'spatial_temporal',\n",
67+
" 'base': 'spatial_temporal_4_8-16-16-32_32-32-64-64_7293',\n",
68+
" 'name': 'spatial_temporal_4_8-16-16-32_32-32-64-64_7293_1',\n",
69+
" 'base_regression': 'spatial_temporal_r_4_8-16-16-32_32-32-64-64_7293',\n",
70+
" 'nlayers': 8,\n",
71+
" 'max_nlayers': 8,\n",
72+
" 'max_act': 14, #this can be manually adjusted as the maximum in the preferred direction histogram\n",
73+
" 'control': False,\n",
74+
" 'cmap': matplotlib.colors.ListedColormap(['midnightblue']),\n",
75+
" 'color': 'midnightblue',\n",
76+
" 'regression_color': 'darkturquoise',\n",
77+
" 'control_cmap': 'Greys_r',\n",
78+
" 'regression_cmap': matplotlib.colors.ListedColormap(['darkturquoise']),\n",
79+
" 's_stride': 2,\n",
80+
" 't_stride': 3,\n",
81+
" 'regression_task': False,\n",
82+
" 'model_path': None,\n",
83+
" 'exp_id': None,})\n",
6584
"\n",
66-
"runinfo = RunInfo({'expid': 102, #internal experiment id\n",
67-
" 'datafraction': 0.5,\n",
85+
"runinfo = RunInfo({'expid': 402, #internal experiment id\n",
86+
" 'datafraction': 'auto', #fraction (0,1] or 'auto' (i.e. if you want to run a new analysis but keep the old results that it would otherwise overwrite, increment by 1)\n",
6887
" 'randomseed': 2000,\n",
6988
" 'randomseed_traintest': 42,\n",
7089
" 'dirr2threshold': 0.2,\n",
71-
" 'verbose': 0,\n",
72-
" 'model_experiment_id': 4, #as per Pranav's model generation\n",
73-
" 'basefolder' : '/home/kai/Dropbox/DeepDrawData/analysis-data/', #point to analysis-data folder\n",
74-
" #in DeepDrawData folder from Dropbox, include trailing slash\n",
75-
" 'orientation' : 'hor',\n",
76-
" 'height' : 'all'\n",
90+
" 'verbose': 2, #0 (least), 1, 2 (most)\n",
91+
" 'model_experiment_id': 22, #used in model training, int or 'auto'\n",
92+
" 'basefolder': '/media/data/DeepDraw/revisions/analysis-data/', ## change this folder to redirect to where the data is saved locally\n",
93+
" 'batchsize': 100, #for layer representation generation\n",
94+
" 'default_run': True, #only variable that is 'trial'-dependent,\n",
95+
" #ie should be changed when rerunning stuff in same folder\n",
96+
" #not semantically important for run info\n",
97+
" 'dpi': 500,\n",
98+
" 'orientation': 'hor',\n",
99+
" 'height': 'all',\n",
77100
" })"
78101
]
79102
},
@@ -86,7 +109,7 @@
86109
},
87110
{
88111
"cell_type": "code",
89-
"execution_count": 59,
112+
"execution_count": 14,
90113
"metadata": {},
91114
"outputs": [],
92115
"source": [
@@ -103,7 +126,7 @@
103126
},
104127
{
105128
"cell_type": "code",
106-
"execution_count": 60,
129+
"execution_count": 15,
107130
"metadata": {},
108131
"outputs": [],
109132
"source": [
@@ -116,15 +139,17 @@
116139
},
117140
{
118141
"cell_type": "code",
119-
"execution_count": 61,
142+
"execution_count": 16,
120143
"metadata": {},
121144
"outputs": [
122145
{
123146
"name": "stdout",
124147
"output_type": "stream",
125148
"text": [
126-
"/mnt/data/random_controls/exp102/data/spatial_temporal_4_8-16-16-32_64-64-64-64_5272/spatial_temporal_4_8-16-16-32_64-64-64-64_5272_1\n",
127-
"using alternate method for accessing kinvars files by directly accessing needed arrays (pandas causes error)\n"
149+
"/media/data/DeepDraw/revisions/analysis-data/exp402/data/spatial_temporal_4_8-16-16-32_32-32-64-64_7293/spatial_temporal_4_8-16-16-32_32-32-64-64_7293_1\n",
150+
"using alternate method for accessing kinvars files by directly accessing needed arrays (pandas causes error)\n",
151+
"1256960\n",
152+
"(1964, 2, 320)\n"
128153
]
129154
}
130155
],
@@ -141,26 +166,36 @@
141166
},
142167
{
143168
"cell_type": "code",
144-
"execution_count": 62,
169+
"execution_count": 17,
145170
"metadata": {},
146-
"outputs": [],
171+
"outputs": [
172+
{
173+
"name": "stdout",
174+
"output_type": "stream",
175+
"text": [
176+
"read layer represenations. shape: (4000, 4, 320, 16)\n"
177+
]
178+
}
179+
],
147180
"source": [
148-
"lo = pickle.load(open(os.path.join(runinfo.datafolder(modelinfo), layer + '.pkl'), 'rb'))\n",
181+
"lo = read_layer_reps(ilayer, runinfo, modelinfo)\n",
149182
"lo = lo[xyplmvt]\n",
150183
"\n",
151-
"centers = get_centers(lo.shape[2])\n",
184+
"centers = get_centers(lo.shape[2], ilayer, modelinfo)\n",
152185
"\n",
153-
"if (fset == 'vel' or fset == 'acc' or fset == 'eepolar' or fset == 'ang' or fset=='angvel' or fset=='ee'):\n",
186+
"if (fset == 'vel' or fset == 'acc' or fset == 'eepolar' or fset == 'ang' or fset=='angvel' or fset=='ee' or fset == 'dir'):\n",
154187
" nmods = 4\n",
155188
" nmets = 6\n",
156189
"elif (fset == 'labels'):\n",
157190
" nmods = 1\n",
158-
" nmets = 1"
191+
" nmets = 1\n",
192+
"else:\n",
193+
" assert False, 'invalid f_set'"
159194
]
160195
},
161196
{
162197
"cell_type": "code",
163-
"execution_count": 63,
198+
"execution_count": 18,
164199
"metadata": {},
165200
"outputs": [],
166201
"source": [
@@ -172,7 +207,7 @@
172207
},
173208
{
174209
"cell_type": "code",
175-
"execution_count": 64,
210+
"execution_count": 19,
176211
"metadata": {},
177212
"outputs": [],
178213
"source": [
@@ -185,7 +220,7 @@
185220
},
186221
{
187222
"cell_type": "code",
188-
"execution_count": 65,
223+
"execution_count": 20,
189224
"metadata": {},
190225
"outputs": [],
191226
"source": [
@@ -323,7 +358,7 @@
323358
},
324359
{
325360
"cell_type": "code",
326-
"execution_count": 66,
361+
"execution_count": 21,
327362
"metadata": {
328363
"scrolled": true
329364
},
@@ -442,7 +477,7 @@
442477
},
443478
{
444479
"cell_type": "code",
445-
"execution_count": 67,
480+
"execution_count": 22,
446481
"metadata": {},
447482
"outputs": [],
448483
"source": [
@@ -465,36 +500,40 @@
465500
},
466501
{
467502
"cell_type": "code",
468-
"execution_count": 69,
503+
"execution_count": 23,
469504
"metadata": {},
470505
"outputs": [
471506
{
472507
"name": "stdout",
473508
"output_type": "stream",
474509
"text": [
475-
"spatial_temporal_4_8-16-16-32_64-64-64-64_5272_1 Layer 3: \n",
510+
"spatial_temporal_4_8-16-16-32_32-32-64-64_7293_1 Layer 3: \n",
476511
"\n",
477512
"RMSE: \n",
478-
"Min: -0.0085\n",
479-
"Max: 0.0389\n",
480-
"Mean: 0.0097\n",
481-
"25pc Quantile: 0.0046\n",
482-
"Median: 0.0093\n",
483-
"75pc Quantile: 0.0127\n",
513+
"Min: nan\n",
514+
"Max: nan\n",
515+
"Mean: nan\n",
516+
"25pc Quantile: nan\n",
517+
"Median: nan\n",
518+
"75pc Quantile: nan\n",
484519
"\n",
485520
"\n",
486-
"R2: \n"
487-
]
488-
},
489-
{
490-
"ename": "IndexError",
491-
"evalue": "index 1 is out of bounds for axis 1 with size 1",
492-
"output_type": "error",
493-
"traceback": [
494-
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
495-
"\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
496-
"\u001b[0;32m<ipython-input-69-fcb51a5e9d9b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\\nR2: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mprint_statistics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtestevals\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\\nPCC: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
497-
"\u001b[0;31mIndexError\u001b[0m: index 1 is out of bounds for axis 1 with size 1"
521+
"R2: \n",
522+
"Min: 0.0000\n",
523+
"Max: 2382973486990906079431681999784995771544749166977351114241131154861388307168959570808358856727180433693371174601667748650793646381061578975953292557346001152289641613754368.0000\n",
524+
"Mean: 6205660122372151248520005207773426488397784289003518526669612382451532049919165548980101189393699046076487433858509762111441787450681195249878366034755211334087608369152.0000\n",
525+
"25pc Quantile: 0.0000\n",
526+
"Median: 0.0000\n",
527+
"75pc Quantile: 0.0000\n",
528+
"\n",
529+
"\n",
530+
"PCC: \n",
531+
"Min: 0.0000\n",
532+
"Max: 0.0000\n",
533+
"Mean: 0.0000\n",
534+
"25pc Quantile: 0.0000\n",
535+
"Median: 0.0000\n",
536+
"75pc Quantile: 0.0000\n"
498537
]
499538
}
500539
],
@@ -538,7 +577,7 @@
538577
"name": "python",
539578
"nbconvert_exporter": "python",
540579
"pygments_lexer": "ipython3",
541-
"version": "3.7.1"
580+
"version": "3.8.5"
542581
}
543582
},
544583
"nbformat": 4,

0 commit comments

Comments
 (0)