From d9ccc92294708627762cef99ee3b80bc9b725ec9 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski <69353402+project-defiant@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:33:01 +0000 Subject: [PATCH] orchestration, pis: move the gwas catalog top hits release into a dated prod run folder The 2026-08-28 top hits run now lives in gs://gwas_catalog_top_hits/prod/20260828T165117/ with its outputs under output/, its DAG config under config/ and its docs under docs/. - gwas_catalog_top_hits.yaml: output_path points at the run's output/ folder - pis/config.yaml: GCCA study index and credible sets read from the run folder - docs: split the top hits docs into gwas_catalog_top_hits/README.md, with the inputs, the gentroutils sync, gentropy links pinned to v3.4.0-rc.3 and the output schemas; regenerate the broken DAG svg - Makefile: upload-gwas-catalog-top-hits-readme uploads only the top hits docs to the run folder Co-Authored-By: Claude Opus 5.5 --- orchestration/Makefile | 8 +- .../datasources/gwas_catalog_data/README.md | 42 +---- .../gwas_catalog_top_hits.svg | 68 -------- .../gwas_catalog_top_hits/README.md | 155 ++++++++++++++++++ .../gwas_catalog_top_hits.svg | 68 ++++++++ .../dags/config/gwas_catalog_top_hits.yaml | 2 +- pis/config.yaml | 4 +- 7 files changed, 239 insertions(+), 108 deletions(-) delete mode 100644 orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits.svg create mode 100644 orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/README.md create mode 100644 orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits.svg diff --git a/orchestration/Makefile b/orchestration/Makefile index 0129d79e3..32a6a193d 100644 --- a/orchestration/Makefile +++ b/orchestration/Makefile @@ -73,7 +73,7 @@ build-dag-svgs: ## Generate visual representations of Airflow DAGs for documenta datasources/gwas_catalog_data/gwas_catalog_sumstats_pics \ datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_clumping \ datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_finemapping \ - datasources/gwas_catalog_data/gwas_catalog_top_hits \ + datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits \ datasources/gwas_catalog_data/gwas_catalog_heritability_estimate \ datasources/gnomad_data/gnomad_ingestion \ datasources/ukb_ppp_eur_data/ukb_ppp_eur_finemapping \ @@ -100,9 +100,12 @@ upload-gwas-catalog-buckets-readme: ## Upload gwas_catalog readme to the bucket( @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_inputs/docs @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_pics/docs @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_susie/docs - @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_top_hits/docs @gcloud storage rsync docs/credible_set_qc gs://gwas_catalog_sumstats_susie/docs/credible_set_qc +GWAS_CATALOG_TOP_HITS_RUN ?= prod/20260828T165117 +upload-gwas-catalog-top-hits-readme: ## Upload the gwas_catalog_top_hits readme to the run folder of its bucket + @gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_top_hits gs://gwas_catalog_top_hits/$(GWAS_CATALOG_TOP_HITS_RUN)/docs + upload-gnomad-bucket-readme: ## Upload gnomad_data readme to the bucket @gcloud storage rsync docs/datasources/gnomad_data gs://gnomad_data_2/docs @@ -119,5 +122,6 @@ upload-decode-bucket-readme: ## Upload decode_data readme to the bucket(s) update-bucket-docs: ## Upload all datasource readmes to their buckets update-bucket-docs: upload-eqtl-catalogue-bucket-readme upload-ukb-ppp-bucket-readme \ upload-finngen-bucket-readme upload-gwas-catalog-buckets-readme \ + upload-gwas-catalog-top-hits-readme \ upload-gnomad-bucket-readme upload-intervals-bucket-readme \ upload-finngen-meta-readme upload-decode-bucket-readme diff --git a/orchestration/docs/datasources/gwas_catalog_data/README.md b/orchestration/docs/datasources/gwas_catalog_data/README.md index 6a542cd17..7064ff260 100644 --- a/orchestration/docs/datasources/gwas_catalog_data/README.md +++ b/orchestration/docs/datasources/gwas_catalog_data/README.md @@ -1,6 +1,6 @@ # GWAS Catalog data source -This document was updated on 2026-01-30. +This document was updated on 2026-10-26. Data stored under 4 buckets: @@ -249,39 +249,7 @@ The image in the `/images/gentropy/Dockerfile` is based on the [gentropy image]( ## GWAS Catalog top hits -Bucket `gs://gwas_catalog_top_hits` contains: - -```sh -gs://gwas_catalog_top_hits/credible_sets/ -gs://gwas_catalog_top_hits/docs/ -gs://gwas_catalog_top_hits/study_index/ -gs://gwas_catalog_top_hits/study_locus_ld_clumped/ -gs://gwas_catalog_top_hits/study_locus_window_based_clumped/ -``` - -Data contained in the `top_hits` corresponds to the [GWAS Catalog top associations](https://www.ebi.ac.uk/gwas/docs/methods/criteria#:~:text=statistics%20here.-,Top%20associations,-We%20also%20report) - -Data is produced by **gwas_catalog_top_hits DAG**. The dag configuration and topology can be found in `gwas_catalog_top_hits.yaml` file under the config directory. - -DAG contains 3 steps: - -![gwas_catalog_top_hits](gwas_catalog_top_hits.svg) - -1. gwas_catalog_top_hits_ingestion -2. ld_based_clumping -3. pics - -### gwas_catalog_top_hits_ingestion - -The step runs on the dataproc cluster that is used to generate the [study_index dataset](https://opentargets.github.io/gentropy/python_api/datasets/study_index/) and perform [window_based_clumping](https://opentargets.github.io/gentropy/python_api/methods/clumping/) on top associations. The datasets are saved under `gs://gwas_catalog_top_hits/study_index/` and `gs://gwas_catalog_top_hits/study_locus_window_based_clumped/`. - -### ld_based_clumping - -The step that performs [ld_clumping](https://opentargets.github.io/gentropy/python_api/methods/clumping/) on the results from the previous step. The results from this step are saved under `gs://gwas_catalog_top_hits/study_locus_ld_clumped/`. - -### pics - -The step that performs [PICS finemapping](https://opentargets.github.io/gentropy/python_api/methods/pics/) based on the ld clumped study locus dataset to obtain the dataset containing credible sets. This dataset is stored under `gs://gwas_catalog_top_hits/credible_sets/` +Produced by the `gwas_catalog_top_hits` DAG into `gs://gwas_catalog_top_hits`, see [GWAS Catalog top hits](gwas_catalog_top_hits/README.md). --- @@ -483,4 +451,8 @@ To adjust the parameters for google batch infrastructure refer to the `google_ba - [chore: 2026/09 GWAS Catalog manual curation update](https://github.com/opentargets/issues/issues/4427) - [chore: 2026/09 GCCA & GCCS pics run](https://github.com/opentargets/issues/issues/4430) - [chore: 2026/09 GCSS susie and credible set qc run](https://github.com/opentargets/issues/issues/4429) -- [chore: 2026/09 GWAS Catalog heritability estimate](https://github.com/opentargets/issues/issues/4432) \ No newline at end of file +- [chore: 2026/09 GWAS Catalog heritability estimate](https://github.com/opentargets/issues/issues/4432) + +### 2026-10-26 + +- chore: move the top hits release (run of 2026-08-28) from the root of `gs://gwas_catalog_top_hits` to `gs://gwas_catalog_top_hits/prod/20260828T165117/` (`config/`, `docs/`, `output/`), and split its docs into [GWAS Catalog top hits](gwas_catalog_top_hits/README.md). diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits.svg b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits.svg deleted file mode 100644 index a52c9ad18..000000000 --- a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits.svg +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - -gwas_catalog_top_hits - -gwas_catalog_top_hits - - -create_cluster - -create_cluster - - - -gwas_catalog_top_hit_ingestion - -gwas_catalog_top_hit_ingestion - - - -create_cluster_otg-tophit-gwascatalog->gwas_catalog_top_hit_ingestion - - - - - -delete_cluster - -delete_cluster - - - -ld_based_clumping - -ld_based_clumping - - - -gwas_catalog_top_hit_ingestion->ld_based_clumping - - - - - -pics - -pics - - - -ld_based_clumping->pics - - - - - -pics->delete_cluster_otg-tophit-gwascatalog - - - - - diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/README.md b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/README.md new file mode 100644 index 000000000..25e5323fb --- /dev/null +++ b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/README.md @@ -0,0 +1,155 @@ +# GWAS Catalog top hits + +This document was updated on 2026-10-26. + +```sh +gs://gwas_catalog_top_hits/prod/20260828T165117/config/gwas_catalog_top_hits.yaml # DAG config of the run +gs://gwas_catalog_top_hits/prod/20260828T165117/docs/ +gs://gwas_catalog_top_hits/prod/20260828T165117/output/credible_sets/ +gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_index/ +gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_locus_ld_clumped/ +gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_locus_window_based_clumped/ +``` + +Data contained in the `top_hits` corresponds to the [GWAS Catalog top associations](https://www.ebi.ac.uk/gwas/docs/methods/criteria#:~:text=statistics%20here.-,Top%20associations,-We%20also%20report) + +Data is produced by **gwas_catalog_top_hits DAG**. The dag configuration and topology can be found in `config/gwas_catalog_top_hits.yaml` of the run folder (`gs://gwas_catalog_top_hits/prod/20260828T165117/config/`). It runs gentropy [`v3.4.0-rc.3`](https://github.com/opentargets/gentropy/tree/v3.4.0-rc.3) (`GENTROPY_REF`), which the links below refer to. + +The run read: + +```sh +gs://gwas_catalog_inputs/gentroutils/20260802/gwas_catalog_download_studies.tsv +gs://gwas_catalog_inputs/gentroutils/20260802/gwas_catalog_download_ancestries.tsv +gs://gwas_catalog_inputs/gentroutils/20260802/gwas_catalog_associations_ontology_annotated.tsv +gs://gnomad_data_2/v4.1/variant_index/ +gs://gnomad_data_2/v2.1.1/ld_index/ +``` + +The three GWAS Catalog files are synced from the [EBI FTP](https://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/) by [gentroutils](https://github.com/opentargets/gentroutils), outside this DAG: studies (`gwas_catalog_download_studies.tsv`), their ancestries (`gwas_catalog_download_ancestries.tsv`) and the curated associations with their ontology annotations (`gwas_catalog_associations_ontology_annotated.tsv`). Each sync is written to a folder named after the date of the GWAS Catalog release, `gentroutils//`. +DAG contains 3 steps: + +![gwas_catalog_top_hits](gwas_catalog_top_hits.svg) + +1. gwas_catalog_top_hit_ingestion +2. ld_based_clumping +3. pics + +## gwas_catalog_top_hit_ingestion + +The step ([`gwas_catalog_top_hits.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/gwas_catalog_top_hits.py)) runs on the dataproc cluster that is used to generate the [study_index dataset](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/dataset/study_index.py) and perform [window_based_clumping](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/window_based_clumping.py) on top associations. The datasets are saved under `gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_index/` and `gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_locus_window_based_clumped/`. + +## ld_based_clumping + +The step ([`ld_based_clumping.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/ld_based_clumping.py)) that performs [ld_clumping](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/clump.py) on the results from the previous step. The results from this step are saved under `gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_locus_ld_clumped/`. + +## pics + +The step ([`pics.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/pics.py)) that performs [PICS finemapping](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/pics.py) based on the ld clumped study locus dataset to obtain the dataset containing credible sets. This dataset is stored under `gs://gwas_catalog_top_hits/prod/20260828T165117/output/credible_sets/` + +## Outputs + +Schemas as read from the parquet files of the run. Nested columns show their `array>` types. + +### study_index + +| Column | Type | Description | +|---|---|---| +| `studyId` | `string` | GWAS Catalog study accession (`GCST…`). | +| `projectId` | `string` | Source of the study: `GCST` | +| `studyType` | `string` | Type of the study: `gwas`. | +| `pubmedId` | `string` | PubMed id of the publication. | +| `publicationFirstAuthor` | `string` | First author of the publication. | +| `publicationDate` | `string` | Publication date. | +| `publicationJournal` | `string` | Journal of the publication. | +| `publicationTitle` | `string` | Title of the publication. | +| `traitFromSource` | `string` | Trait as reported by the GWAS Catalog. | +| `initialSampleSize` | `string` | Discovery sample as reported, in free text. | +| `traitFromSourceMappedIds` | `array` | Ontology (EFO) terms the trait is mapped to. Populated by GWAS Catalog. | +| `backgroundTraitFromSourceMappedIds` | `array` | Ontology (EFO) terms of the background trait shared by all samples, e.g. a disease the cohort has. | +| `cohorts` | `array` | Cohorts of the study, as reported. | +| `discoverySamples` | `array>` | Ancestry and size of each discovery sample. | +| `ldPopulationStructure` | `array>` | LD populations of the discovery samples and their share of the sample; the largest one is the LD reference of the study. | +| `replicationSamples` | `array>` | Ancestry and size of each replication sample. | +| `nCases` | `integer` | Number of cases. | +| `nControls` | `integer` | Number of controls. | +| `nSamples` | `integer` | Number of samples. | +| `qualityControls` | `array` | Quality control flags of the study. | + +### study_locus_window_based_clumped + +| Column | Type | Description | +|---|---|---| +| `studyId` | `string` | GWAS Catalog study accession (`GCST…`) of the association, as in `study_index`. | +| `studyLocusId` | `string` | Identifier of the study locus: an md5 hash of `studyId` and `variantId`. | +| `variantId` | `string` | Lead variant, `___` on GRCh38, mapped to gnomAD v4.1. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `pValueMantissa` | `float` | Mantissa of the reported p-value (p = mantissa × 10^exponent). | +| `pValueExponent` | `integer` | Exponent of the reported p-value. | +| `qualityControls` | `array` | Quality control flags of the study locus. | +| `beta` | `double` | Effect size harmonised to the alternate allele of `variantId`; reported odds ratios are converted to beta. | +| `standardError` | `double` | Standard error of `beta`, derived from the effect size and the p-value, or from the odds ratio confidence interval. | + +### study_locus_ld_clumped + +| Column | Type | Description | +|---|---|---| +| `variantId` | `string` | Lead variant, `___` on GRCh38, from the gnomAD v4.1 variant index. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `studyId` | `string` | GWAS Catalog study accession (`GCST…`) of the association, as in `study_index`. | +| `studyLocusId` | `string` | Identifier of the study locus: a hash of `studyId` and `variantId`. | +| `studyType` | `string` | Empty in this release. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `region` | `string` | Empty in this release. | +| `beta` | `double` | Effect size harmonised to the alternate allele of `variantId`; reported odds ratios are converted to beta. | +| `zScore` | `double` | Empty in this release. | +| `pValueMantissa` | `float` | Mantissa of the reported p-value (p = mantissa × 10^exponent). | +| `pValueExponent` | `integer` | Exponent of the reported p-value. | +| `effectAlleleFrequencyFromSource` | `float` | Empty in this release. | +| `standardError` | `double` | Standard error of `beta`, derived from the effect size and the p-value, or from the odds ratio confidence interval. | +| `subStudyDescription` | `string` | Empty in this release. | +| `qualityControls` | `array` | Quality control flags of the study locus. | +| `finemappingMethod` | `string` | Empty until `pics`. | +| `credibleSetIndex` | `integer` | Empty in this release. | +| `credibleSetlog10BF` | `double` | Empty in this release. | +| `purityMeanR2` | `double` | Empty in this release. | +| `purityMinR2` | `double` | Empty in this release. | +| `locusStart` | `integer` | Empty in this release. | +| `locusEnd` | `integer` | Empty in this release. | +| `sampleSize` | `integer` | Empty in this release. | +| `locus` | `array>` | Empty until `pics`. | +| `confidence` | `string` | Empty in this release. | +| `isTransQtl` | `boolean` | Empty in this release. | +| `ldSet` | `array>` | Variants in LD with the lead (`tagVariantId`) and their r² (`r2Overall`) in the major LD population of the study; only r² of at least 0.5. | + +### credible_sets + +| Column | Type | Description | +|---|---|---| +| `studyLocusId` | `string` | Identifier of the study locus: a hash of `studyId` and `variantId`. | +| `studyType` | `string` | Empty in this release. | +| `variantId` | `string` | Lead variant, `___` on GRCh38, from the gnomAD v4.1 variant index. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `region` | `string` | Empty in this release. | +| `studyId` | `string` | GWAS Catalog study accession (`GCST…`) of the association, as in `study_index`. | +| `beta` | `double` | Effect size harmonised to the alternate allele of `variantId`; reported odds ratios are converted to beta. | +| `zScore` | `double` | Empty in this release. | +| `pValueMantissa` | `float` | Mantissa of the reported p-value (p = mantissa × 10^exponent). | +| `pValueExponent` | `integer` | Exponent of the reported p-value. | +| `effectAlleleFrequencyFromSource` | `float` | Empty in this release. | +| `standardError` | `double` | Standard error of `beta`, derived from the effect size and the p-value, or from the odds ratio confidence interval. | +| `subStudyDescription` | `string` | Empty in this release. | +| `qualityControls` | `array` | Quality control flags of the study locus. | +| `finemappingMethod` | `string` | Fine-mapping method: `PICS`. | +| `credibleSetIndex` | `integer` | Empty in this release. | +| `credibleSetlog10BF` | `double` | Empty in this release. | +| `purityMeanR2` | `double` | Empty in this release. | +| `purityMinR2` | `double` | Empty in this release. | +| `locusStart` | `integer` | Empty in this release. | +| `locusEnd` | `integer` | Empty in this release. | +| `sampleSize` | `integer` | Empty in this release. | +| `ldSet` | `array>` | Variants in LD with the lead (`tagVariantId`) and their r² (`r2Overall`) in the major LD population of the study; only r² of at least 0.5. | +| `confidence` | `string` | Empty in this release. | +| `isTransQtl` | `boolean` | Empty in this release. | +| `locus` | `array>` | Variants of the credible set: `variantId`, its r² with the lead (`r2Overall`), its `posteriorProbability`, and whether it is in the 95% and 99% credible sets (`is95CredibleSet`, `is99CredibleSet`). | diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits.svg b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits.svg new file mode 100644 index 000000000..12e8d166f --- /dev/null +++ b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits.svg @@ -0,0 +1,68 @@ + + + + + + +gwas_catalog_top_hits + +gwas_catalog_top_hits + + +create_cluster_otg-tophit-gwascatalog + +create_cluster_otg-tophit-gwascatalog + + + +gwas_catalog_top_hit_ingestion + +gwas_catalog_top_hit_ingestion + + + +create_cluster_otg-tophit-gwascatalog->gwas_catalog_top_hit_ingestion + + + + + +delete_cluster_otg-tophit-gwascatalog + +delete_cluster_otg-tophit-gwascatalog + + + +ld_based_clumping + +ld_based_clumping + + + +gwas_catalog_top_hit_ingestion->ld_based_clumping + + + + + +pics + +pics + + + +ld_based_clumping->pics + + + + + +pics->delete_cluster_otg-tophit-gwascatalog + + + + + diff --git a/orchestration/src/orchestration/dags/config/gwas_catalog_top_hits.yaml b/orchestration/src/orchestration/dags/config/gwas_catalog_top_hits.yaml index d906f924c..6eafd4b1c 100644 --- a/orchestration/src/orchestration/dags/config/gwas_catalog_top_hits.yaml +++ b/orchestration/src/orchestration/dags/config/gwas_catalog_top_hits.yaml @@ -3,7 +3,7 @@ environment_specs: - name: Prod vars: input_path: gs://gwas_catalog_inputs - output_path: gs://gwas_catalog_top_hits + output_path: gs://gwas_catalog_top_hits/prod/20260828T165117/output sync_date: '20260802' gnomad_path: gs://gnomad_data_2 diff --git a/pis/config.yaml b/pis/config.yaml index fcba69f4e..78fa51cfc 100644 --- a/pis/config.yaml +++ b/pis/config.yaml @@ -682,13 +682,13 @@ steps: #: GWAS STEP :###################################################################################### gwas: - name: copy_many GCCA study - sources: gs://gwas_catalog_top_hits/study_index/*.parquet + sources: gs://gwas_catalog_top_hits/prod/20260828T165117/output/study_index/*.parquet settings: billing_project: open-targets-eu-dev destination: input/gwas/study/gcca - name: copy_many GCCA credible sets - sources: gs://gwas_catalog_top_hits/credible_sets/*.parquet + sources: gs://gwas_catalog_top_hits/prod/20260828T165117/output/credible_sets/*.parquet settings: billing_project: open-targets-eu-dev destination: input/gwas/credible_set/gcca