V1DD Dataset

import pandas as pd 
import numpy as np
from datetime import datetime, date

from aind_data_access_api.document_db import MetadataDbClient

API_GATEWAY_HOST = "api.allenneuraldynamics.org"
DATABASE = 'metadata_index'
COLLECTION = 'data_assets'

docdb_api_client = MetadataDbClient(
   host=API_GATEWAY_HOST,
    version="v2",
   database=DATABASE,
   collection=COLLECTION,
)
print(docdb_api_client._base_url)

https://api.allenneuraldynamics.org/v2/metadata_index/data_assets

aggregate = [
  {
    "$match": {
      "data_description.project_name": {
        "$regex": "V1 Deep Dive",
        "$options": "i"
      },
      "name": {
        "$regex": "filtered",
        "$options": "i"
      },
      "location": {
        "$regex": "aind-open-data",
        "$options": "i"
      }
    }
  },
  {
    "$project": {
      "name": 1, 
      "subject_id": "$data_description.subject_id",
      "genotype": "$subject.subject_details.genotype", 
      "date_of_birth": "$subject.subject_details.date_of_birth", 
      "sex": "$subject.subject_details.sex", 
      "session_time": "$acquisition.acquisition_start_time",
      "project_name": "$data_description.project_name", 
      "modality": "$data_description.modalities.name",
      "column": { "$arrayElemAt": ["$data_description.tags", 0] },
      "volume": { "$arrayElemAt": ["$data_description.tags", 1] }
    }
  },
]
    
records = docdb_api_client.aggregate_docdb_records(
    pipeline = aggregate,
)

We can turn these records into a dataframe and reorganize some things:

df = pd.DataFrame(records)

df['session_date'] = df.apply(lambda x: datetime.fromisoformat(x['session_time']).date(), axis=1)
df['session_time'] = df.apply(lambda x: datetime.fromisoformat(x['session_time']).time(), axis=1)
df['date_of_birth'] = df.apply(lambda x: datetime.strptime(x['date_of_birth'], '%Y-%m-%d').date(), axis=1)
df['age'] = df.apply(lambda x: (x['session_date'] - x['date_of_birth']).days, axis=1)

df['column'] = df.apply(lambda x: int(x['column'].split(' ')[-1]), axis=1)
df['volume'] = df.apply(lambda x: int(x['volume'].split(' ')[-1]), axis=1)

df['golden_mouse'] = False
df.loc[df.subject_id=='409828', 'golden_mouse'] = True

order = ['project_name','_id','name','subject_id','golden_mouse','genotype','date_of_birth','sex','modality',
         'session_date','age','session_time','column','volume']
df = df[order]

df.head()

	project_name	_id	name	subject_id	golden_mouse	genotype	date_of_birth	sex	modality	session_date	age	session_time	column	volume
0	V1 Deep Dive	e79d9464-c9b9-4a10-a23c-c1ba21515680	427836_2019-04-24_13-06-45_filtered_2026-04-09...	427836	False	Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai94(TITL-G...	2018-10-08	Female	[Planar optical physiology, Behavior videos]	2019-04-24	198	13:06:45.257460	4	3
1	V1 Deep Dive	483fe6e0-b116-4850-9bb3-8d7251d6d7b3	427836_2019-04-22_14-06-13_filtered_2026-08-07...	427836	False	Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai94(TITL-G...	2018-10-08	Female	[Planar optical physiology, Behavior videos]	2019-04-22	196	14:06:13.779820	2	3
2	V1 Deep Dive	fa1ce50c-deb6-4a18-aa37-ef122d21512d	409828_2018-12-11_14-40-36_filtered_2026-04-09...	409828	True	Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai94(TITL-G...	2018-07-03	Male	[Planar optical physiology, Behavior videos]	2018-12-11	161	14:40:36.449080	4	5
3	V1 Deep Dive	cf88135f-cf4a-4a2b-823d-a184c6f3ca79	427836_2019-02-20_13-54-22_filtered_2026-04-09...	427836	False	Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai94(TITL-G...	2018-10-08	Female	[Planar optical physiology, Behavior videos]	2019-02-20	135	13:54:22.282920	1	2
4	V1 Deep Dive	ecc75433-04ce-40c8-b337-ff0515a4356b	438833_2019-03-08_14-59-45_filtered_2026-04-09...	438833	False	Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai94(TITL-G...	2018-12-01	Male	[Planar optical physiology, Behavior videos]	2019-03-08	97	14:59:45.383070	1	1

print(len(df))

100