orchestration: create a staging cluster when the steps before its first steps end

orchestration/Makefile

@@ -4,8 +4,9 @@ LOCAL_DEV_CREDENTIALS ?= ~/.config/gcloud/application_default_credentials.json
 ### HOUSEKEEPING TARGETS ###
 .PHONY: help clean clean-vm \
         test lint check local-airflow cloud-airflow tunnel \
-				upload-eqtl-catalogue-bucket-readme upload-ukb-ppp-bucket-readme \
-				upload-finngen-bucket-readme upload-gwas-catalog-buckets-readme \
+				upload-eqtl-catalogue-readme upload-ukb-ppp-bucket-readme \
+				upload-finngen-gwas-readme upload-finngen-multiome-readme \
+				upload-gwas-catalog-buckets-readme \
 				upload-gnomad-bucket-readme upload-intervals-bucket-readme \
 				upload-finngen-meta-readme upload-decode-bucket-readme
 
@@ -70,7 +71,7 @@ build-dag-svgs: ## Generate visual representations of Airflow DAGs for documenta
 	@AIRFLOW__CORE__DAGS_FOLDER=src/orchestration/dags uv run --frozen airflow db migrate
 	@AIRFLOW__CORE__DAGS_FOLDER=src/orchestration/dags uv run --frozen airflow dags reserialize
 	@$(foreach dag, \
-		datasources/gwas_catalog_data/gwas_catalog_sumstats_pics \
+		datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/gwas_catalog_sumstats_pics \
 		datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_clumping \
 		datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_finemapping \
 		datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits \
@@ -79,32 +80,42 @@ build-dag-svgs: ## Generate visual representations of Airflow DAGs for documenta
 		datasources/ukb_ppp_eur_data/ukb_ppp_eur_finemapping \
 		datasources/ukb_ppp_eur_data/ukb_ppp_eur_harmonisation \
 		datasources/finngen_meta_data/finngen_ukb_mvp_meta \
-		datasources/finngen_data/finngen_ingestion \
+		datasources/finngen_data/finngen_gwas/finngen_ingestion \
+		datasources/finngen_data/finngen_multiome/finngen_multiome_ingestion \
 		datasources/eqtl_catalogue_data/eqtl_catalogue_ingestion \
 		datasources/decode_data/decode_ingestion \
 		credible_set_qc/credible_set_qc \
 		unified_pipeline/unified_pipeline, \
 		AIRFLOW__CORE__LOAD_EXAMPLES=false AIRFLOW__CORE__DAGS_FOLDER=src/orchestration/dags uv run --frozen airflow dags show --save docs/$(dag).svg $(notdir $(dag));)
 
-upload-eqtl-catalogue-bucket-readme: ## Upload eqtl_catalogue_data readme to the bucket
-	@gcloud storage rsync docs/datasources/eqtl_catalogue_data gs://eqtl_catalogue_data/docs
+upload-eqtl-catalogue-readme: ## Upload the eqtl_catalogue_data readme to a run folder: make upload-eqtl-catalogue-readme BUCKET=gs://eqtl_catalogue_data/r8/prod/<YYYYMMDDTHHMMSS>
+	$(if $(BUCKET),,$(error BUCKET is not set: make upload-eqtl-catalogue-readme BUCKET=gs://eqtl_catalogue_data/r8/prod/<YYYYMMDDTHHMMSS>))
+	@gcloud storage rsync docs/datasources/eqtl_catalogue_data $(BUCKET)/docs
 
 upload-ukb-ppp-bucket-readme: ## Upload ukb_ppp_eur_data readme to the bucket
 	@gcloud storage rsync docs/datasources/ukb_ppp_eur_data gs://ukb_ppp_eur_data/docs
 	@gcloud storage rsync docs/credible_set_qc gs://ukb_ppp_eur_data/docs/credible_set_qc
 
-upload-finngen-bucket-readme: ## Upload finngen_data readme to the bucket
-	@gcloud storage rsync docs/datasources/finngen_data gs://finngen_data/docs
+upload-finngen-gwas-readme: ## Upload the finngen gwas readme to a run folder: make upload-finngen-gwas-readme BUCKET=gs://finngen_data/gwas/prod/<YYYYMMDDTHHMMSS>
+	$(if $(BUCKET),,$(error BUCKET is not set: make upload-finngen-gwas-readme BUCKET=gs://finngen_data/gwas/prod/<YYYYMMDDTHHMMSS>))
+	@gcloud storage rsync docs/datasources/finngen_data/finngen_gwas $(BUCKET)/docs
+
+upload-finngen-multiome-readme: ## Upload the finngen multiome readme to a run folder: make upload-finngen-multiome-readme BUCKET=gs://finngen_data/multiome/prod/<YYYYMMDDTHHMMSS>
+	$(if $(BUCKET),,$(error BUCKET is not set: make upload-finngen-multiome-readme BUCKET=gs://finngen_data/multiome/prod/<YYYYMMDDTHHMMSS>))
+	@gcloud storage rsync docs/datasources/finngen_data/finngen_multiome $(BUCKET)/docs
 
 upload-gwas-catalog-buckets-readme: ## Upload gwas_catalog readme to the bucket(s)
 	@gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_inputs/docs
-	@gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_pics/docs
 	@gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_susie/docs
 	@gcloud storage rsync docs/credible_set_qc gs://gwas_catalog_sumstats_susie/docs/credible_set_qc
 
-GWAS_CATALOG_TOP_HITS_RUN ?= prod/20260828T165117
-upload-gwas-catalog-top-hits-readme: ## Upload the gwas_catalog_top_hits readme to the run folder of its bucket
-	@gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_top_hits gs://gwas_catalog_top_hits/$(GWAS_CATALOG_TOP_HITS_RUN)/docs
+upload-gwas-catalog-top-hits-readme: ## Upload the gwas_catalog_top_hits readme to a run folder: make upload-gwas-catalog-top-hits-readme BUCKET=gs://gwas_catalog_top_hits/prod/<YYYYMMDDTHHMMSS>
+	$(if $(BUCKET),,$(error BUCKET is not set: make upload-gwas-catalog-top-hits-readme BUCKET=gs://gwas_catalog_top_hits/prod/<YYYYMMDDTHHMMSS>))
+	@gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_top_hits $(BUCKET)/docs
+
+upload-gwas-catalog-sumstats-pics-readme: ## Upload the gwas_catalog_sumstats_pics readme to a run folder: make upload-gwas-catalog-sumstats-pics-readme BUCKET=gs://gwas_catalog_sumstats_pics/prod/<YYYYMMDDTHHMMSS>
+	$(if $(BUCKET),,$(error BUCKET is not set: make upload-gwas-catalog-sumstats-pics-readme BUCKET=gs://gwas_catalog_sumstats_pics/prod/<YYYYMMDDTHHMMSS>))
+	@gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics $(BUCKET)/docs
 
 upload-gnomad-bucket-readme: ## Upload gnomad_data readme to the bucket
 	@gcloud storage rsync docs/datasources/gnomad_data gs://gnomad_data_2/docs
@@ -120,8 +131,7 @@ upload-decode-bucket-readme: ## Upload decode_data readme to the bucket(s)
 	@gcloud storage rsync docs/datasources/decode_data gs://decode_inputs/docs
 
 update-bucket-docs: ## Upload all datasource readmes to their buckets
-update-bucket-docs: upload-eqtl-catalogue-bucket-readme upload-ukb-ppp-bucket-readme \
-									  upload-finngen-bucket-readme upload-gwas-catalog-buckets-readme \
-										upload-gwas-catalog-top-hits-readme \
+update-bucket-docs: upload-ukb-ppp-bucket-readme \
+									  upload-gwas-catalog-buckets-readme \
 										upload-gnomad-bucket-readme upload-intervals-bucket-readme \
 										upload-finngen-meta-readme upload-decode-bucket-readme

orchestration/docs/datasources/bigbrain_data/README.md

@@ -0,0 +1,155 @@
+# BigBrain
+
+This document was updated on 2026-10-29.
+
+```sh
+gs://bigbrain_data/prod/20261029T162943/config/bigbrain_ingestion.yaml # DAG config of the run 20261029T125742
+gs://bigbrain_data/prod/20261029T162943/config/patch_publication.py  # patch of pubmedId and initialSampleSize
+gs://bigbrain_data/prod/20261029T162943/docs/     # this file
+gs://bigbrain_data/prod/20261029T162943/output/eqtl/study/
+gs://bigbrain_data/prod/20261029T162943/output/eqtl/credible_set/
+gs://bigbrain_data/prod/20261029T162943/output/eqtl/credible_set_variant/
+gs://bigbrain_data/prod/20261029T162943/output/sqtl/study/
+gs://bigbrain_data/prod/20261029T162943/output/sqtl/credible_set/
+gs://bigbrain_data/prod/20261029T162943/output/sqtl/credible_set_variant/
+```
+
+Use `prod/20261029T162943/`. It is the DAG run `prod/20261029T125742/` with `pubmedId` and `initialSampleSize` of the two study indexes patched (see [Patch of 2026-10-29](#patch-of-2026-10-29)); its credible sets are the same files. `prod/20261029T125742/` keeps the DAG output unchanged, with its `config/`, `bin/` and `source/`.
+
+[BigBrain](https://zenodo.org/records/17226890) is a brain-tissue molecular QTL meta-analysis (Réal, Kailash et al. 2025, "Mapping genetic effects on splicing in ten thousand post-mortem brain samples reveals novel mediators of neurological disease risk"). Its published SuSiE fine-mapping of the European (EUR) release is ingested as credible sets. It was run by BigBrain with PolyFun + SuSiE (L=5, 1 Mb window around the feature) on Z-scores, with in-sample LD from 3,884 European donors. The credible sets are not fine-mapped again.
+
+Data is produced by the **bigbrain_ingestion DAG**. The DAG configuration is in `config/bigbrain_ingestion.yaml` of the run folder. The run uses gentropy [`v4.0.0-dev.5`](https://github.com/opentargets/gentropy/tree/v4.0.0-dev.5) and the step [`bigbrain_finemapping_ingestion`](https://github.com/opentargets/gentropy/blob/v4.0.0-dev.5/src/gentropy/bigbrain_ingestion.py) ([gentropy#1321](https://github.com/opentargets/gentropy/pull/1321)). Each run writes to its own folder `gs://bigbrain_data/prod/<YYYYMMDDTHHMMSS>/`.
+
+Inputs of the run:
+
+```sh
+gs://bigbrain_data/prod/20261029T125742/source/BigBrain_cis_eQTL_finemapping.txt.gz     # Zenodo record 17226890
+gs://bigbrain_data/prod/20261029T125742/source/BigBrain_cis_sQTL_finemapping.txt.gz     # Zenodo record 17153730
+gs://bigbrain_data/prod/20261029T125742/source/BigBrain_cis_sQTL_EUR_top_assoc.tsv.gz   # Zenodo record 17153730, junction-to-gene mapping
+gs://open-targets-pipeline-runs/do/platform-2610-2/output/target/                    # target index, sQTL only
+```
+
+The DAG has 3 steps:
+
+1. `download`: copies the three BigBrain files from [record 17226890](https://zenodo.org/records/17226890) and [record 17153730](https://zenodo.org/records/17153730) into `source/`.
+2. `eqtl`: one study per gene. Writes `output/eqtl/`.
+3. `sqtl`: one study per intron junction (Leafcutter). Writes `output/sqtl/`.
+
+`eqtl` and `sqtl` run in parallel on one Dataproc cluster.
+
+## Contents of the run 20261029T125742
+
+| | Studies | Credible sets | Credible set variants |
+|---|---:|---:|---:|
+| eQTL | 16,691 | 42,082 | 296,754 |
+| sQTL | 35,879 | 68,808 | 654,666 |
+
+- Each `output/<qtl>/<dataset>/` is a folder of Parquet files.
+- `credible_set` is keyed by `id`. `credible_set_variant` joins to it on `credibleSetId`, and `credible_set` joins to `study` on `studyId`.
+- Every credible set has a study and at least one variant. `id` is unique.
+- 3,121 sQTL studies have no `geneId`: the junction has no gene, or it has several genes and none could be chosen (see `study`).
+
+## How the source is converted
+
+From [`finemapping.py`](https://github.com/opentargets/gentropy/blob/v4.0.0-dev.5/src/gentropy/datasource/bigbrain/finemapping.py) and [`study_index.py`](https://github.com/opentargets/gentropy/blob/v4.0.0-dev.5/src/gentropy/datasource/bigbrain/study_index.py):
+
+- The published credible sets are kept as they are. PolyFun reports the SuSiE 95% credible sets.
+- `variantId` is `CHR_BP_A2_A1`, so the effect allele `A1` is the alternate allele.
+- Rows with `Z = 0` are untested placeholder variants. Credible sets led by one (about 2%) are dropped, and other such rows are removed.
+- `posteriorProbability` is the published PIP scaled to sum to 0.9501 per credible set. The published PIP is the probability of being causal in any SuSiE effect of the feature, so the raw sums range from 0.03 to 2.6. Scaling keeps the order of the members.
+- The lead variant is the one with the top PIP, ties broken by `variantId`. `locusStart` and `locusEnd` are the positions of the first and last member.
+- P-values are recomputed from `Z`.
+- Bayes factors are not published, so they are null. Use these sets with PIP-based colocalisation (`coloc_pip_ecaviar`).
+- Purity is not published. It is 1 for single-variant sets and null for others. SuSiE only reports sets with a minimum absolute correlation of 0.5, so all sets passed a purity filter at source. Credible set QC that drops null purity would drop most sets.
+- `conditionalBeta` and `conditionalStandardError` are the SuSiE posterior mean and standard deviation (`BETA_MEAN`, `BETA_SD`), summed over all effects of the feature and on the standardised scale. Use them for the direction of effect only, not for effect size. For low-PIP members the sign can come from another effect. No marginal beta is published.
+- sQTL: a junction annotated to several genes (about 10%) is resolved with the target index to the one gene that contains it, or else to the one protein-coding gene that contains it. Otherwise `geneId` is null.
+
+## Outputs
+
+### study
+
+One row per gene (eQTL) or intron junction (sQTL).
+
+| Column | Type | Description |
+|---|---|---|
+| `studyId` | `string` | `BigBrain_<eqtl\|sqtl>_EUR_<feature>`, e.g. `BigBrain_eqtl_EUR_ENSG00000173627.8`, `BigBrain_sqtl_EUR_chr1_54668937_54670799`. |
+| `geneId` | `string` | Ensembl gene id without version. sQTL: null for 3,121 junctions. |
+| `traitFromSource` | `string` | eQTL: versioned Ensembl gene id. sQTL: gene symbols of the junction from `top_assoc`, joined by `;`; null for 144 junctions. |
+| `projectId` | `string` | `BigBrain`. |
+| `studyType` | `string` | `eqtl` or `sqtl`. |
+| `biosampleFromSourceId` | `string` | `UBERON_0000955` (brain). |
+| `pubmedId` | `string` | `41040686`, the BigBrain paper (medRxiv preprint). |
+| `publicationFirstAuthor` | `string` | `Réal A, Kailash BP`. |
+| `publicationDate` | `string` | `2025`. |
+| `publicationTitle` | `string` | Title of the BigBrain paper. |
+| `initialSampleSize` | `string` | `10,725 samples from 4,656 donors (all ancestries)`: the whole resource, not split by ancestry. |
+| `nSamples` | `integer` | `10725`, as above. |
+| `discoverySamples` | `array<struct<sampleSize: integer, ancestry: string>>` | `[{10725, European}]`. |
+| `ldPopulationStructure` | `array<struct<ldPopulation: string, relativeSampleSize: double>>` | `[{nfe, 1.0}]`. |
+| `cohorts` | `array<string>` | `[BigBrain]`. |
+| `summarystatsLocation` | `string` | Zenodo URL of the full summary statistics of the QTL type. They are not ingested. |
+| `hasSumstats` | `boolean` | `true`. |
+
+### credible_set
+
+One row per credible set.
+
+| Column | Type | Description |
+|---|---|---|
+| `id` | `string` | Identifier of the credible set. |
+| `studyId` | `string` | Study of the credible set, as in `study`. |
+| `studyType` | `string` | `eqtl` or `sqtl`. |
+| `chromosome` | `string` | Chromosome, `1` to `22`. |
+| `locusStart` | `integer` | Position of the first member variant on GRCh38. |
+| `locusEnd` | `integer` | Position of the last member variant on GRCh38. |
+| `leadVariantId` | `string` | Member with the top PIP, `<chromosome>_<position>_<ref>_<alt>` on GRCh38. |
+| `fineMappingMethod` | `string` | `SuSiE`. |
+| `logBayesFactor` | `double` | Null. |
+| `confidence` | `tinyint` | Null. |
+| `purityMeanR2` | `double` | 1 for single-variant sets (eQTL 16,018, sQTL 25,912), null otherwise. |
+| `purityMinR2` | `double` | As `purityMeanR2`. |
+| `qualityControls` | `array<string>` | Empty. |
+
+### credible_set_variant
+
+One row per member variant of a credible set.
+
+| Column | Type | Description |
+|---|---|---|
+| `credibleSetId` | `string` | `id` of the credible set. |
+| `variantId` | `string` | `<chromosome>_<position>_<ref>_<alt>` on GRCh38. |
+| `posteriorProbability` | `float` | Published PIP, scaled to sum to 0.9501 per credible set. |
+| `pValueMantissa` | `float` | Mantissa of the p-value from `Z` (p = mantissa × 10^exponent). |
+| `pValueExponent` | `integer` | Exponent of the p-value. |
+| `sampleSize` | `integer` | `N` of the association test at the variant. |
+| `conditionalBeta` | `float` | SuSiE posterior mean (`BETA_MEAN`), summed over effects, standardised scale. Direction only. |
+| `conditionalStandardError` | `float` | SuSiE posterior standard deviation (`BETA_SD`), as above. |
+| `effectAlleleFrequency` | `float` | Null. |
+| `reportedBeta` | `double` | Null. |
+| `reportedStandardError` | `double` | Null. |
+| `logBayesFactor` | `float` | Null. |
+| `r2Overall` | `float` | Null. |
+| `rescaledAbsBeta` | `float` | Null. |
+| `rescaledStandardError` | `float` | Null. |
+| `rescaledDirection` | `tinyint` | Null. |
+| `qualityControls` | `array<string>` | Empty. |
+
+## Patch of 2026-10-29
+
+gentropy `v4.0.0-dev.5` set `pubmedId` to `35058635`, the mmQTL method paper, and `initialSampleSize` to `10,725 donors (EUR)`. The BigBrain paper is PubMed [41040686](https://pubmed.ncbi.nlm.nih.gov/41040686/), and it reports 10,725 samples from 4,656 donors of all ancestries. `config/patch_publication.py` sets both columns in `output/{eqtl,sqtl}/study/` with pyarrow. Other columns, the column types and the row order are unchanged. The gentropy fix is [gentropy#1333](https://github.com/opentargets/gentropy/pull/1333).
+
+Run as `python patch_publication.py <study of 20261029T125742>/part-00000-<id>.zstd.parquet part-00000.zstd.parquet`, once for `eqtl` and once for `sqtl`.
+
+## Other runs
+
+- `gs://bigbrain_data/prod/20261029T125742/`: the DAG run, before the patch.
+
+- `gs://bigbrain_data/prod/20261015T103255/`: an earlier run from the gentropy#1321 branch (commit `777abba9`), with the eQTL and sQTL credible sets together in `output/credible_set/` and `output/study_index/`. Same counts. See its `docs/README.md`.
+
+## Changelog
+
+### 2026-10-29
+
+- feat: add the `bigbrain_ingestion` DAG, a StagingDagConfig DAG on gentropy `v4.0.0-dev.5`.
+- docs: describe the outputs of the run `20261029T125742`; the same file is in `docs/` of the run folder.
+- fix: patch `pubmedId` and `initialSampleSize` of the study indexes into `gs://bigbrain_data/prod/20261029T162943/`.