diff --git a/orchestration/Makefile b/orchestration/Makefile index 32a6a193d..17118f0f9 100644 --- a/orchestration/Makefile +++ b/orchestration/Makefile @@ -70,7 +70,7 @@ build-dag-svgs: ## Generate visual representations of Airflow DAGs for documenta @AIRFLOW__CORE__DAGS_FOLDER=src/orchestration/dags uv run --frozen airflow db migrate @AIRFLOW__CORE__DAGS_FOLDER=src/orchestration/dags uv run --frozen airflow dags reserialize @$(foreach dag, \ - datasources/gwas_catalog_data/gwas_catalog_sumstats_pics \ + datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/gwas_catalog_sumstats_pics \ datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_clumping \ datasources/gwas_catalog_data/gwas_catalog_sumstats_susie_finemapping \ datasources/gwas_catalog_data/gwas_catalog_top_hits/gwas_catalog_top_hits \ @@ -98,7 +98,6 @@ upload-finngen-bucket-readme: ## Upload finngen_data readme to the bucket upload-gwas-catalog-buckets-readme: ## Upload gwas_catalog readme to the bucket(s) @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_inputs/docs - @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_pics/docs @gcloud storage rsync docs/datasources/gwas_catalog_data gs://gwas_catalog_sumstats_susie/docs @gcloud storage rsync docs/credible_set_qc gs://gwas_catalog_sumstats_susie/docs/credible_set_qc @@ -106,6 +105,10 @@ GWAS_CATALOG_TOP_HITS_RUN ?= prod/20260828T165117 upload-gwas-catalog-top-hits-readme: ## Upload the gwas_catalog_top_hits readme to the run folder of its bucket @gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_top_hits gs://gwas_catalog_top_hits/$(GWAS_CATALOG_TOP_HITS_RUN)/docs +upload-gwas-catalog-sumstats-pics-readme: ## Upload the gwas_catalog_sumstats_pics readme to a run folder: make upload-gwas-catalog-sumstats-pics-readme BUCKET=gs://gwas_catalog_sumstats_pics/prod/ + $(if $(BUCKET),,$(error BUCKET is not set: make upload-gwas-catalog-sumstats-pics-readme BUCKET=gs://gwas_catalog_sumstats_pics/prod/)) + @gcloud storage rsync docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics $(BUCKET)/docs + upload-gnomad-bucket-readme: ## Upload gnomad_data readme to the bucket @gcloud storage rsync docs/datasources/gnomad_data gs://gnomad_data_2/docs diff --git a/orchestration/docs/datasources/gwas_catalog_data/README.md b/orchestration/docs/datasources/gwas_catalog_data/README.md index 7064ff260..7f363b31b 100644 --- a/orchestration/docs/datasources/gwas_catalog_data/README.md +++ b/orchestration/docs/datasources/gwas_catalog_data/README.md @@ -255,43 +255,9 @@ Produced by the `gwas_catalog_top_hits` DAG into `gs://gwas_catalog_top_hits`, s ## GWAS Catalog pics summary statistics -Bucket `gs://gwas_catalog_sumstats_pics` contains: +Produced by the `gwas_catalog_sumstats_pics` DAG into `gs://gwas_catalog_sumstats_pics`, see [GWAS Catalog summary statistics PICS](gwas_catalog_sumstats_pics/README.md). -```sh -gs://gwas_catalog_sumstats_pics/credible_sets/ -gs://gwas_catalog_sumstats_pics/study_index/ -gs://gwas_catalog_sumstats_pics/study_locus_ld_clumped/ -gs://gwas_catalog_sumstats_pics/study_locus_window_based_clumped/ -``` - -Data contained in the `pics summary statistics` corresponds to the GWAS Catalog summary statistics harmonised to the Open Targets [SummaryStatistics dataset](https://opentargets.github.io/gentropy/python_api/datasets/summary_statistics/). - -Data is produced by **gwas_catalog_sumstats_pics DAG**. The dag configuration and topology can be found in `gwas_catalog_sumstats_pics.yaml` file under the config directory. - -DAG contains 4 steps: - -![gwas_catalog_sumstats_pics](gwas_catalog_sumstats_pics.svg) - -1. gwas_catalog_study_index -2. window_based_clumping -3. ld_based_clumping -4. pics - -### gwas_catalog_study_index - -The step runs on the dataproc cluster that is used to generate the [study_index dataset](https://opentargets.github.io/gentropy/python_api/datasets/study_index/) that is saved under the `gs://gwas_catalog_sumstats_pics/study_index/` - -### window_based_clumping - -The step that performs the [window_based_clumping](https://opentargets.github.io/gentropy/python_api/methods/clumping/) on harmonised summary statistics. The dataset is saved under `gs://gs://gwas_catalog_sumstats_pics/study_locus_ld_clumped/`. - -### ld_based_clumping - sumstats - -The step that performs [ld_clumping](https://opentargets.github.io/gentropy/python_api/methods/clumping/) on the results from window based clumping step. The results from this step are saved under `gs://gwas_catalog_top_hits/study_locus_ld_clumped/`. - -### pics - sumstats - -The step that performs [PICS finemapping](https://opentargets.github.io/gentropy/python_api/methods/pics/) based on the ld clumped study locus dataset to obtain the dataset containing credible sets. This dataset is stored under `gs://gwas_catalog_sumstats_pics/credible_sets/` +--- ## GWAS Catalog susie summary statistics @@ -456,3 +422,7 @@ To adjust the parameters for google batch infrastructure refer to the `google_ba ### 2026-10-26 - chore: move the top hits release (run of 2026-08-28) from the root of `gs://gwas_catalog_top_hits` to `gs://gwas_catalog_top_hits/prod/20260828T165117/` (`config/`, `docs/`, `output/`), and split its docs into [GWAS Catalog top hits](gwas_catalog_top_hits/README.md). + +### 2026-10-29 + +- chore: move the summary statistics PICS release (run of 2026-08-28) from the root of `gs://gwas_catalog_sumstats_pics` to `gs://gwas_catalog_sumstats_pics/prod/20260828T115527/` (`config/`, `docs/`, `output/`), and split its docs into [GWAS Catalog summary statistics PICS](gwas_catalog_sumstats_pics/README.md). diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics.svg b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics.svg deleted file mode 100644 index 3cefa4012..000000000 --- a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics.svg +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - -gwas_catalog_sumstats_pics - -gwas_catalog_sumstats_pics - - -create_cluster - -create_cluster - - - -gwas_catalog_study_index - -gwas_catalog_study_index - - - -create_cluster_otg-gwascatalog-sumstats-pics->gwas_catalog_study_index - - - - - -window_based_clumping - -window_based_clumping - - - -create_cluster_otg-gwascatalog-sumstats-pics->window_based_clumping - - - - - -delete_cluster - -delete_cluster - - - -gwas_catalog_study_index->delete_cluster_otg-gwascatalog-sumstats-pics - - - - - -ld_based_clumping - -ld_based_clumping - - - -pics - -pics - - - -ld_based_clumping->pics - - - - - -pics->delete_cluster_otg-gwascatalog-sumstats-pics - - - - - -window_based_clumping->ld_based_clumping - - - - - diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/README.md b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/README.md new file mode 100644 index 000000000..2625a866f --- /dev/null +++ b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/README.md @@ -0,0 +1,165 @@ +# GWAS Catalog summary statistics PICS + +This document was updated on 2026-10-29. + +```sh +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/config/gwas_catalog_sumstats_pics.yaml # DAG config of the run +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/docs/ +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/study_index/ +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/study_locus_window_based_clumped/ +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/study_locus_ld_clumped/ +gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/credible_sets/ +``` + +Data contained in the `sumstats_pics` corresponds to the GWAS Catalog summary statistics harmonised to the Open Targets [SummaryStatistics dataset](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/dataset/summary_statistics.py), fine-mapped with PICS. + +Data is produced by **gwas_catalog_sumstats_pics DAG**. The DAG configuration and topology are in `config/gwas_catalog_sumstats_pics.yaml` of the run folder (`gs://gwas_catalog_sumstats_pics/prod/20260828T115527/config/`). The run uses gentropy [`v3.4.0-rc.3`](https://github.com/opentargets/gentropy/tree/v3.4.0-rc.3) (`GENTROPY_REF`); the links below refer to that tag. + +Inputs of the run: + +```sh +gs://gwas_catalog_inputs/gentroutils/20260802/gwas_catalog_download_studies.tsv +gs://gwas_catalog_inputs/gentroutils/20260802/gwas_catalog_download_ancestries.tsv +gs://gwas_catalog_inputs/harmonised_summary_statistics/ +gs://gnomad_data_2/v2.1.1/ld_index/ +``` + +The two GWAS Catalog files are synced from the [EBI FTP](https://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/) by [gentroutils](https://github.com/opentargets/gentroutils), outside this DAG: studies (`gwas_catalog_download_studies.tsv`) and their ancestries (`gwas_catalog_download_ancestries.tsv`). Each sync is written to a folder named after the date of the GWAS Catalog release, `gentroutils//`. The harmonised summary statistics are the output of the `gwas_catalog_harmonisation` DAG, one folder per study (`harmonised_summary_statistics//`); see the [GWAS Catalog inputs](../README.md#gwas-catalog-inputs). + +The DAG has 4 steps, run on one Dataproc cluster: + +![gwas_catalog_sumstats_pics](gwas_catalog_sumstats_pics.svg) + +1. gwas_catalog_study_index +2. window_based_clumping +3. ld_based_clumping +4. pics + +## gwas_catalog_study_index + +The step ([`gwas_catalog_study_index.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/gwas_catalog_study_index.py)) builds one study per GWAS Catalog study from the studies and ancestries files ([`study_index.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/datasource/gwas_catalog/study_index.py)). The study index covers every study of the GWAS Catalog release, with or without summary statistics, and carries no curation or summary statistics QC annotation. The output is `output/study_index/`. + +## window_based_clumping + +The step ([`window_based_clumping.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/window_based_clumping.py)) reads all harmonised summary statistics and keeps, per study, the variants with a p-value of at most 1e-8 that are the most significant within 500 kb ([`window_based_clumping.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/window_based_clumping.py)). Each kept variant is the lead of one study locus. The output is `output/study_locus_window_based_clumped/`. + +## ld_based_clumping + +The step ([`ld_based_clumping.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/ld_based_clumping.py)) adds to each study locus the variants in LD with its lead, from the gnomAD v2.1.1 LD index, in the major LD population of the study. A study locus whose lead is in LD with the lead of a more significant study locus of the same study is flagged `Explained by a more significant variant in high LD` and its LD set is emptied ([`clump.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/clump.py)). A study locus whose lead is missing from the LD index is flagged `Variant not found in LD reference`, and its LD set holds the lead only. The output is `output/study_locus_ld_clumped/`. + +## pics + +The step ([`pics.py`](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/pics.py)) fine-maps each study locus with [PICS](https://github.com/opentargets/gentropy/blob/v3.4.0-rc.3/src/gentropy/method/pics.py), from the p-value of the lead and its LD set. Every study locus, flagged or not, gives one row, flagged `Study locus finemapped without in-sample LD reference`; a study locus with an empty LD set gives an empty credible set. The output is `output/credible_sets/`. + +## Outputs + +### study_index + +| Column | Type | Description | +|---|---|---| +| `studyId` | `string` | GWAS Catalog study accession (`GCST…`). | +| `projectId` | `string` | Source of the study: `GCST`. | +| `studyType` | `string` | Type of the study: `gwas`. | +| `pubmedId` | `string` | PubMed id of the publication. | +| `publicationFirstAuthor` | `string` | First author of the publication. | +| `publicationDate` | `string` | Publication date. | +| `publicationJournal` | `string` | Journal of the publication. | +| `publicationTitle` | `string` | Title of the publication. | +| `traitFromSource` | `string` | Trait as reported by the GWAS Catalog. | +| `initialSampleSize` | `string` | Discovery sample as reported, in free text. | +| `traitFromSourceMappedIds` | `array` | Ontology (EFO) terms the trait is mapped to by the GWAS Catalog. | +| `backgroundTraitFromSourceMappedIds` | `array` | Ontology (EFO) terms of the background trait shared by all samples, e.g. a disease of the cohort. | +| `cohorts` | `array` | Cohorts of the study, as reported. | +| `discoverySamples` | `array>` | Ancestry and size of each discovery sample. | +| `ldPopulationStructure` | `array>` | LD populations of the discovery samples and their share of the sample; the largest one is the LD population of the study. | +| `replicationSamples` | `array>` | Ancestry and size of each replication sample. | +| `nCases` | `integer` | Number of cases. | +| `nControls` | `integer` | Number of controls. | +| `nSamples` | `integer` | Number of samples. | + +### study_locus_window_based_clumped + +One row per lead. + +| Column | Type | Description | +|---|---|---| +| `studyId` | `string` | Study of the summary statistics, as in `study_index`. | +| `variantId` | `string` | Lead variant, `___` on GRCh38. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `beta` | `double` | Effect size of the lead, harmonised to the alternate allele. | +| `sampleSize` | `integer` | Sample size of the lead, where reported. | +| `pValueMantissa` | `float` | Mantissa of the p-value (p = mantissa × 10^exponent). | +| `pValueExponent` | `integer` | Exponent of the p-value. | +| `effectAlleleFrequencyFromSource` | `float` | Frequency of the effect allele, where reported. | +| `standardError` | `double` | Standard error of `beta`. | +| `studyLocusId` | `string` | Identifier of the study locus: a hash of `studyId` and `variantId`. | +| `qualityControls` | `array` | Empty. | + +### study_locus_ld_clumped + +The rows of `study_locus_window_based_clumped`, with the LD flags of `ld_based_clumping`. + +| Column | Type | Description | +|---|---|---| +| `variantId` | `string` | Lead variant, `___` on GRCh38. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `studyId` | `string` | Study of the study locus, as in `study_index`. | +| `studyLocusId` | `string` | Identifier of the study locus: a hash of `studyId` and `variantId`. | +| `studyType` | `string` | Empty. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `region` | `string` | Empty. | +| `beta` | `double` | Effect size of the lead, harmonised to the alternate allele. | +| `zScore` | `double` | Empty. | +| `pValueMantissa` | `float` | Mantissa of the p-value of the lead. | +| `pValueExponent` | `integer` | Exponent of the p-value of the lead. | +| `effectAlleleFrequencyFromSource` | `float` | Frequency of the effect allele of the lead, where reported. | +| `standardError` | `double` | Standard error of `beta`. | +| `subStudyDescription` | `string` | Empty. | +| `qualityControls` | `array` | Quality control flags of the study locus. | +| `finemappingMethod` | `string` | Empty. | +| `credibleSetIndex` | `integer` | Empty. | +| `credibleSetlog10BF` | `double` | Empty. | +| `purityMeanR2` | `double` | Empty. | +| `purityMinR2` | `double` | Empty. | +| `locusStart` | `integer` | Empty. | +| `locusEnd` | `integer` | Empty. | +| `sampleSize` | `integer` | Sample size of the lead, where reported. | +| `locus` | `array>` | Empty. | +| `confidence` | `string` | Empty. | +| `isTransQtl` | `boolean` | Empty. | +| `ldSet` | `array>` | Variants in LD with the lead (`tagVariantId`) and their r² with it (`r2Overall`) in the major LD population of the study. Empty for a study locus flagged `Explained by a more significant variant in high LD`. | + +### credible_sets + +One row per study locus. + +| Column | Type | Description | +|---|---|---| +| `studyLocusId` | `string` | Identifier of the study locus: a hash of `studyId` and `variantId`. | +| `studyType` | `string` | Empty. | +| `variantId` | `string` | Lead variant, `___` on GRCh38. | +| `chromosome` | `string` | Chromosome of the lead variant. | +| `position` | `integer` | Position of the lead variant on GRCh38. | +| `region` | `string` | Empty. | +| `studyId` | `string` | Study of the credible set, as in `study_index`. | +| `beta` | `double` | Effect size of the lead, harmonised to the alternate allele. | +| `zScore` | `double` | Empty. | +| `pValueMantissa` | `float` | Mantissa of the p-value of the lead. | +| `pValueExponent` | `integer` | Exponent of the p-value of the lead. | +| `effectAlleleFrequencyFromSource` | `float` | Frequency of the effect allele of the lead, where reported. | +| `standardError` | `double` | Standard error of `beta`. | +| `subStudyDescription` | `string` | Empty. | +| `qualityControls` | `array` | Quality control flags of the study locus, plus `Study locus finemapped without in-sample LD reference`. | +| `finemappingMethod` | `string` | Fine-mapping method: `PICS`. | +| `credibleSetIndex` | `integer` | Empty. | +| `credibleSetlog10BF` | `double` | Empty. | +| `purityMeanR2` | `double` | Empty. | +| `purityMinR2` | `double` | Empty. | +| `locusStart` | `integer` | Empty. | +| `locusEnd` | `integer` | Empty. | +| `sampleSize` | `integer` | Sample size of the lead, where reported. | +| `ldSet` | `array>` | As in `study_locus_ld_clumped`. | +| `confidence` | `string` | Empty. | +| `isTransQtl` | `boolean` | Empty. | +| `locus` | `array>` | Variants of the 99% credible set: `variantId`, its r² with the lead (`r2Overall`), its PICS `posteriorProbability`, and whether it is in the 95% and 99% credible sets. `pValueMantissa`, `pValueExponent`, `beta` and `standardError` are set for the lead only. Empty for a study locus with an empty `ldSet`. | diff --git a/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/gwas_catalog_sumstats_pics.svg b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/gwas_catalog_sumstats_pics.svg new file mode 100644 index 000000000..fd749d025 --- /dev/null +++ b/orchestration/docs/datasources/gwas_catalog_data/gwas_catalog_sumstats_pics/gwas_catalog_sumstats_pics.svg @@ -0,0 +1,86 @@ + + + + + + +gwas_catalog_sumstats_pics + +gwas_catalog_sumstats_pics + + +create_cluster_otg-gwascatalog-sumstats-pics + +create_cluster_otg-gwascatalog-sumstats-pics + + + +gwas_catalog_study_index + +gwas_catalog_study_index + + + +create_cluster_otg-gwascatalog-sumstats-pics->gwas_catalog_study_index + + + + + +window_based_clumping + +window_based_clumping + + + +create_cluster_otg-gwascatalog-sumstats-pics->window_based_clumping + + + + + +delete_cluster_otg-gwascatalog-sumstats-pics + +delete_cluster_otg-gwascatalog-sumstats-pics + + + +gwas_catalog_study_index->delete_cluster_otg-gwascatalog-sumstats-pics + + + + + +ld_based_clumping + +ld_based_clumping + + + +pics + +pics + + + +ld_based_clumping->pics + + + + + +pics->delete_cluster_otg-gwascatalog-sumstats-pics + + + + + +window_based_clumping->ld_based_clumping + + + + + diff --git a/orchestration/src/orchestration/dags/config/gwas_catalog_sumstats_pics.yaml b/orchestration/src/orchestration/dags/config/gwas_catalog_sumstats_pics.yaml index 362c16c26..2a0180221 100644 --- a/orchestration/src/orchestration/dags/config/gwas_catalog_sumstats_pics.yaml +++ b/orchestration/src/orchestration/dags/config/gwas_catalog_sumstats_pics.yaml @@ -3,7 +3,7 @@ environment_specs: - name: Prod vars: input_path: gs://gwas_catalog_inputs - output_path: gs://gwas_catalog_sumstats_pics + output_path: gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output sync_date: '20260802' env: Prod diff --git a/pis/config.yaml b/pis/config.yaml index 78fa51cfc..56e1c8160 100644 --- a/pis/config.yaml +++ b/pis/config.yaml @@ -694,13 +694,13 @@ steps: destination: input/gwas/credible_set/gcca - name: copy_many GCSS pics study - sources: gs://gwas_catalog_sumstats_pics/study_index/*.parquet + sources: gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/study_index/*.parquet settings: billing_project: open-targets-eu-dev destination: input/gwas/study/gcss_pics - name: copy_many GCSS pics credible sets - sources: gs://gwas_catalog_sumstats_pics/credible_sets/*.parquet + sources: gs://gwas_catalog_sumstats_pics/prod/20260828T115527/output/credible_sets/*.parquet settings: billing_project: open-targets-eu-dev destination: input/gwas/credible_set/gcss_pics