-
Notifications
You must be signed in to change notification settings - Fork 76
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* trying again with pre-commit * rearranging TOC * removed asdf file * fixing last line scec_tutorials * clearing outputs from scoped_download * fix notebook commit --------- Co-authored-by: Yiyu Ni <niyiyu@uw.edu>
- Loading branch information
Showing
5 changed files
with
170 additions
and
21 deletions.
There are no files selected for viewing
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# Download S3-hosted Noisepy Data\n", | ||
| "\n", | ||
| "This notebook is designed to query cross-corelations data calculated by noisepy, hosted on S3, and downloaded locally.\n", | ||
| "\n", | ||
| "This notebook assumes that you have installed the noisepy package. It installs Python tools for MongoDB, queries our SCOPED data base, and parse the S3-hosted data into the ASDF H5 data format." | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "!pip install pymongo" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "from pymongo.mongo_client import MongoClient\n", | ||
| "from pymongo.server_api import ServerApi\n", | ||
| "\n", | ||
| "uri = \"mongodb+srv://user:hG0osHjBdXovsxq8@scoped.nfcjw38.mongodb.net/?retryWrites=true&w=majority\"\n", | ||
| "\n", | ||
| "# Create a new client and connect to the server\n", | ||
| "client = MongoClient(uri, server_api=ServerApi('1'))\n", | ||
| "\n", | ||
| "# Send a ping to confirm a successful connection\n", | ||
| "try:\n", | ||
| " client.admin.command('ping')\n", | ||
| " print(\"Pinged your deployment. You successfully connected to MongoDB!\")\n", | ||
| "except Exception as e:\n", | ||
| " print(e)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "db = client.scoped_noise\n", | ||
| "station_pair_collection = db.station_pair\n", | ||
| "station_collection = db.station\n", | ||
| "\n", | ||
| "# Query the first 10 records in the station_pair collection\n", | ||
| "station_pair_records = station_pair_collection.find().limit(10)\n", | ||
| "\n", | ||
| "print(\"Station Pair Collection:\")\n", | ||
| "for record in station_pair_records:\n", | ||
| " print(record)\n", | ||
| "\n", | ||
| "# Query the first 10 records in the station collection\n", | ||
| "station_records = station_collection.find().limit(10)\n", | ||
| "\n", | ||
| "print(\"\\nStation Collection:\")\n", | ||
| "for record in station_records:\n", | ||
| " print(record)\n", | ||
| " sta_source = record[\"name\"]\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import os\n", | ||
| "import noisepy\n", | ||
| "from noisepy.seis.asdfstore import ASDFStackStore\n", | ||
| "from noisepy.seis.numpystore import NumpyStackStore\n", | ||
| "import time as time\n", | ||
| "\n", | ||
| "stack_data_path = \"s3://scoped-noise/scedc_CI_2022_stack/\"\n", | ||
| "S3_STORAGE_OPTIONS = {\"s3\": {\"anon\": False}}\n", | ||
| "stack_store = NumpyStackStore(stack_data_path, storage_options=S3_STORAGE_OPTIONS)\n", | ||
| "\n", | ||
| "# Get list of station pairs (~47k pairs)\n", | ||
| "t0=time.time()\n", | ||
| "pairs = stack_store.get_station_pairs()\n", | ||
| "t1=time.time() \n", | ||
| "print(f\"Time to get station pairs: {t1-t0} seconds\")\n", | ||
| "# Get the first timespan available for the first pair\n", | ||
| "t2=time.time()\n", | ||
| "ts = stack_store.get_timespans(*pairs[0])[0]\n", | ||
| "t3=time.time()\n", | ||
| "print(f\"Time to get timespans: {t3-t2} seconds\")\n", | ||
| "print(f\"Timespan: {ts}\")\n", | ||
| "\n", | ||
| "# Read some stacks (10?) from S3/numpy\n", | ||
| "stacks_10 = stack_store.read_bulk(ts, pairs[0:10]) \n", | ||
| "\n", | ||
| "# write them to ASDF\n", | ||
| "output= \"./asdf_data\"\n", | ||
| "os.makedirs(output, exist_ok=True)\n", | ||
| "asdf_store = ASDFStackStore(output)\n", | ||
| "for ((src,rec), stacks) in stacks_10:\n", | ||
| " asdf_store.append(ts, src, rec, stacks)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import pyasdf\n", | ||
| "\n", | ||
| "\n", | ||
| "df = pyasdf.ASDFDataSet(\"./asdf_data/CI.ABL/CI.ABL_CI.ABL.h5\", mode=\"r\")\n", | ||
| "print(df)\n", | ||
| "df.auxiliary_data.Allstack_linear.ZZ.data.shape" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "noisepy", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.10.12" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 2 | ||
| } |