Merge pull request #937 from cmu-delphi/sgratzl/meta2.0

use new metadata and show all signals in dashboard

.github/workflows/create_release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Reset main branch
         run: |
           git fetch origin dev:dev
-          git reset --hard dev
+          git reset --hard origin/dev
       - name: Change version number
         id: version
         run: |

netlify.toml

@@ -3,13 +3,10 @@
 
 [context.branch-deploy.environment]
   COVIDCAST_ENDPOINT_URL = "https://api.covidcast.cmu.edu/epidata"
-  COVIDCAST_ANNOTATION_DRAFTS = "true"
 
 [context.deploy-preview.environment]
   COVIDCAST_ENDPOINT_URL = "https://api.covidcast.cmu.edu/epidata"
-  COVIDCAST_ANNOTATION_DRAFTS = "true"
 
 [context.staging.environment]
-  COVIDCAST_ENDPOINT_URL = "https://api-staging.delphi.cmu.edu/epidata"
-  COVIDCAST_SIGNAL_DOC = "https://docs.google.com/document/d/1llv6xh8jMlmVv7WpyDSv4VgUpFAZQ6QjFUeRxxOinmk,https://docs.google.com/document/d/17GQFgJDHF_ckkn-BE72QaHtyL2Dh09S8-4xk0r6G47w"
-  COVIDCAST_ANNOTATION_DRAFTS = "true"
+  COVIDCAST_ENDPOINT_URL = "https://staging.delphi.cmu.edu/epidata"
+  # COVIDCAST_SIGNAL_DOC = "https://docs.google.com/document/d/1llv6xh8jMlmVv7WpyDSv4VgUpFAZQ6QjFUeRxxOinmk,https://docs.google.com/document/d/17GQFgJDHF_ckkn-BE72QaHtyL2Dh09S8-4xk0r6G47w"

package.json

@@ -15,6 +15,7 @@
   "main": "public/bundle.js",
   "files": [
     "public/**.js",
+    "public/**.map",
     "public/**.txt",
     "public/**.css",
     "!public/wrapper.*"

src/App.svelte

@@ -1,20 +1,32 @@
 <script>
   import { onMount } from 'svelte';
-  import { currentMode, appReady, sensorList, loadAnnotations } from './stores';
+  import { currentMode, appReady, loadMetaData, loadAnnotations } from './stores';
   import './stores/urlHandler';
   import './stores/ga';
   import './stores/websiteInjector';
-  import { loadMetaData } from './data/meta';
   import Disclaimer from './components/Disclaimer.svelte';
 
+  let markReady = () => false;
+  const appReadyPromise = new Promise((resolve) => {
+    markReady = resolve;
+  });
+
   onMount(() => {
-    Promise.all([loadMetaData(sensorList), loadAnnotations()]).then(() => {
+    Promise.all([loadMetaData(), loadAnnotations()]).then(() => {
       appReady.set(true);
+      markReady();
       document.body.dataset.ready = 'ready';
     });
   });
 
-  $: currentComponent = $currentMode.component();
+  async function loadComponent(mode) {
+    if (mode.waitForReady) {
+      await appReadyPromise;
+    }
+    return mode.component();
+  }
+
+  $: currentComponent = loadComponent($currentMode);
 </script>
 
 {#await currentComponent}

src/blocks/HistoryLineChart.svelte

@@ -13,6 +13,7 @@
     MULTI_COLORS,
     genDateHighlight,
     genAnnotationLayer,
+    generateCumulativeBarSpec,
   } from '../specs/lineSpec';
   import { toTimeValue } from '../data/utils';
   import Toggle from '../components/Toggle.svelte';
@@ -41,7 +42,7 @@
    */
   export let sensor;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 
@@ -86,28 +87,40 @@
   };
 
   $: highlightDate = date.value;
-  $: timeFrame = showFull && expandableWindow ? date.sensorTimeFrame : date.windowTimeFrame;
+  $: timeFrame = showFull && expandableWindow ? sensor.timeFrame : date.windowTimeFrame;
 
   /**
    * @param {import('../../stores/params').SensorParam} sensor
    * @param {import('../../stores/params').RegionParam} region
    * @param {import('../../stores/params').DateParam} date
    * @param {import('../../stores/params').TimeFrame} timeFrame
-   * @param {{height: number, zero: boolean, singleRaw: boolean, isMobile: boolean, singleRegionOnly: boolean}} options
+   * @param {{height: number, zero: boolean, raw: boolean, isMobile: boolean, singleRegionOnly: boolean, cumulative: boolean}} options
    */
-  function genSpec(sensor, region, date, timeFrame, { height, zero, singleRaw, isMobile, singleRegionOnly, domain }) {
+  function genSpec(
+    sensor,
+    region,
+    date,
+    timeFrame,
+    { height, zero, raw, isMobile, singleRegionOnly, domain, cumulative },
+  ) {
     const options = {
       initialDate: highlightDate || date.value,
       height,
       color,
       domain: domain || timeFrame.domain,
       zero,
       xTitle: sensor.xAxis,
-      title: joinTitle([sensor.name, `in ${region.displayName}`], isMobile),
+      title: joinTitle([(cumulative ? 'Cumulative ' : '') + sensor.name, `in ${region.displayName}`], isMobile),
       subTitle: sensor.unit,
       highlightRegion: true,
     };
-    if (singleRaw) {
+    if (cumulative) {
+      options.paddingLeft = 52; // more space for larger numbers
+    }
+    if (raw) {
+      if (cumulative) {
+        return generateCumulativeBarSpec(options);
+      }
       return generateLineAndBarSpec(options);
     }
     if (singleRegionOnly) {
@@ -167,16 +180,33 @@
    * @param {import("../../stores/params").DateParam} date
    * @param {import("../../stores/params").RegionParam} region
    */
-  function loadSingleData(sensor, region, timeFrame) {
+  function loadSingleData(sensor, region, timeFrame, { cumulative }) {
     if (!region.value) {
       return null;
     }
-    const selfData = fetcher.fetch1Sensor1RegionNDates(sensor, region, timeFrame);
+
+    const selfData = fetcher.fetch1Sensor1RegionNDates(sensor.value, region, timeFrame);
     const rawData = fetcher.fetch1Sensor1RegionNDates(sensor.rawValue, region, timeFrame);
 
-    return Promise.all([selfData, rawData]).then((data) => {
-      return combineSignals(data, data[0], ['smoothed', 'raw']);
-    });
+    if (cumulative) {
+      // raw and cumulative
+      const cumulativeData = fetcher.fetch1Sensor1RegionNDates(sensor.rawCumulativeValue, region, timeFrame);
+      return Promise.all([selfData, rawData, cumulativeData]).then((data) => {
+        return combineSignals(
+          data,
+          data[0].map((d) => ({ ...d })),
+          ['smoothed', 'raw', 'cumulative'],
+        );
+      });
+    } else {
+      return Promise.all([selfData, rawData]).then((data) => {
+        return combineSignals(
+          data,
+          data[0].map((d) => ({ ...d })),
+          ['smoothed', 'raw'],
+        );
+      });
+    }
   }
 
   function onSignal(event) {
@@ -207,12 +237,15 @@
    * @param {import("../../stores/params").SensorParam} sensor
    * @param {import("../../stores/params").Region[]} region
    */
-  function generateFileName(sensor, regions, timeFrame, raw) {
+  function generateFileName(sensor, regions, timeFrame, raw, cumulative) {
     const regionName = regions.map((region) => `${region.propertyId}-${region.displayName}`).join(',');
     let suffix = '';
     if (raw) {
       suffix = '_RawVsSmoothed';
     }
+    if (cumulative) {
+      suffix += '_Cumulative';
+    }
     return `${sensor.name}_${regionName}_${formatDateISO(timeFrame.min)}-${formatDateISO(timeFrame.max)}${suffix}`;
   }
 
@@ -231,24 +264,29 @@
 
   let zoom = false;
   let singleRaw = false;
+  let singleCumulative = false;
 
+  $: raw = singleRaw && sensor.rawValue != null;
+  $: cumulative = raw && singleCumulative && sensor.rawCumulativeValue != null;
   $: regions = raw ? [region.value] : resolveRegions(region.value, singleRegionOnly);
   $: annotations = $annotationManager.getWindowAnnotations(sensor.value, regions, timeFrame.min, timeFrame.max);
-  $: raw = singleRaw && sensor.rawValue != null;
   $: spec = injectRanges(
     genSpec(sensor, region, date, timeFrame, {
       height,
       zero: !zoom,
-      singleRaw: raw,
+      raw,
       isMobile: $isMobileDevice,
       singleRegionOnly,
       domain,
+      cumulative,
     }),
     timeFrame,
     annotations,
   );
-  $: data = raw ? loadSingleData(sensor, region, timeFrame) : loadData(sensor, region, timeFrame, singleRegionOnly);
-  $: fileName = generateFileName(sensor, regions, timeFrame, raw);
+  $: data = raw
+    ? loadSingleData(sensor, region, timeFrame, { cumulative })
+    : loadData(sensor, region, timeFrame, singleRegionOnly);
+  $: fileName = generateFileName(sensor, regions, timeFrame, raw, cumulative);
 
   function findValue(region, data, date, prop = 'value') {
     if (!date) {
@@ -292,15 +330,18 @@
   <Toggle bind:checked={zoom}>Rescale Y-axis</Toggle>
   {#if sensor.rawValue != null}
     <Toggle bind:checked={singleRaw}>Raw Data</Toggle>
+    {#if raw && sensor.rawCumulativeValue != null}
+      <Toggle bind:checked={singleCumulative}>Cumulative Data</Toggle>
+    {/if}
   {/if}
   {#if expandableWindow}
     <Toggle bind:checked={showFull}>Show All Dates</Toggle>
   {/if}
   <div class="spacer" />
-  <DownloadMenu {fileName} {vegaRef} {data} {sensor} {raw} />
+  <DownloadMenu {fileName} {vegaRef} {data} {sensor} {raw} {cumulative} />
 </div>
 
-<div class="{!(singleRaw && sensor.rawValue != null) && regions.length > 1 ? 'mobile-two-col' : ''} legend">
+<div class="{!raw && regions.length > 1 ? 'mobile-two-col' : ''} legend">
   {#each regions as r, i}
     <div
       class="legend-elem"
@@ -324,9 +365,14 @@
         {#await data then d}
           <span class="legend-value">
             <SensorValue {sensor} value={findValue(r, d, highlightDate)} medium />
-            {#if singleRaw && sensor.rawValue != null}
+            {#if raw}
               (raw:
-              <SensorValue {sensor} value={findValue(r, d, highlightDate, 'raw')} medium />)
+              {#if cumulative}
+                <SensorValue {sensor} value={findValue(r, d, highlightDate, 'raw')} medium />, cumulative:
+                <SensorValue {sensor} value={findValue(r, d, highlightDate, 'cumulative')} medium />)
+              {:else}
+                <SensorValue {sensor} value={findValue(r, d, highlightDate, 'raw')} medium />)
+              {/if}
             {/if}
           </span>
         {/await}

src/blocks/IndicatorOverview.svelte

@@ -23,7 +23,7 @@
    */
   export let region;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 

src/blocks/IndicatorStatsLine.svelte

@@ -15,7 +15,7 @@
    */
   export let region;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 

src/blocks/IndicatorWarning.svelte

@@ -20,7 +20,7 @@
   <div data-uk-alert class="uk-alert-warning">
     The indicator "{sensor.name}" does not support the geographic level: {getLevelInfo(region.level).labelPlural}.
   </div>
-{:else if date.sensorTimeFrame.max < date.value}
+{:else if sensor.timeFrame.max < date.value}
   <div data-uk-alert class="uk-alert-warning">
     The indicator "{sensor.name}" is not available for {formatDateYearDayOfWeekAbbr(date.value)}, yet.
   </div>

src/blocks/MaxDateHint.svelte

@@ -1,21 +1,15 @@
 <script>
   import UIKitHint from '../components/UIKitHint.svelte';
   import { formatDateYearDayOfWeekAbbr } from '../formats';
-  import { stats } from '../stores';
-  import { determineStatsInfo } from '../stores/stats';
+  import { metaDataManager } from '../stores';
   /**
    * @type {import('../stores/params').Sensor}
    */
   export let sensor;
 
-  /**
-   * @type {import('../stores/params').RegionLevel}
-   */
-  export let level = 'nation';
-
   export let suffix = '';
 
-  $: info = determineStatsInfo($stats, sensor, level);
+  $: info = $metaDataManager.getMetaData(sensor);
 </script>
 
 {#if info}

src/blocks/RegionCountyMap.svelte

@@ -3,7 +3,7 @@
   import { formatDateISO } from '../formats';
   import { getStateOfCounty } from '../data/regions';
   import { generateStateMapWithCountyDataSpec } from '../specs/mapSpec';
-  import { isMobileDevice, stats } from '../stores';
+  import { isMobileDevice } from '../stores';
   import DownloadMenu from '../components/DownloadMenu.svelte';
   import RegionMapTooltip from './RegionMapTooltip.svelte';
 
@@ -20,12 +20,12 @@
    */
   export let sensor;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 
   $: spec = generateStateMapWithCountyDataSpec({
-    domain: sensor.domain($stats, 'county'),
+    domain: sensor.domain('county'),
     scheme: sensor.value.vegaColorScale,
   });
   $: data = fetcher.fetch1SensorNRegions1Date(sensor, 'county', date);

src/blocks/RegionHexMap.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { isMobileDevice, stats } from '../stores';
+  import { isMobileDevice } from '../stores';
   import { state2TileCell } from '../specs/matrixSpec';
   import { generateSparkLine } from '../specs/lineSpec';
   import HexGrid from '../components/HexGrid/HexGrid.svelte';
@@ -29,7 +29,7 @@
    */
   export let sensor;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 
@@ -83,7 +83,7 @@
     interactive: false,
     domain: date.sparkLineTimeFrame.domain,
   });
-  $: colorScale = sensor.createColorScale($stats, region.level);
+  $: colorScale = sensor.createColorScale(region.level);
   $: dumpData = Promise.all(tileData.map((d) => d.dump)).then((rows) => rows.flat());
 
   const maxColumn = state2TileCell.reduce((acc, v) => Math.max(acc, v.x), 0) + 1;

src/blocks/RegionMap.svelte

@@ -3,7 +3,7 @@
   import { formatDateISO } from '../formats';
   import { getCountiesOfState, getInfoByName } from '../data/regions';
   import { generateCountiesOfStateSpec, generateRelatedCountySpec, generateStateSpec } from '../specs/mapSpec';
-  import { stats, isMobileDevice } from '../stores';
+  import { isMobileDevice } from '../stores';
   import DownloadMenu from '../components/DownloadMenu.svelte';
   import RegionMapTooltip from './RegionMapTooltip.svelte';
 
@@ -20,17 +20,17 @@
    */
   export let sensor;
   /**
-   * @type {import("../../stores/params").DataFetcher}
+   * @type {import("../../stores/DataFetcher").DataFetcher}
    */
   export let fetcher;
 
   /**
    * @param {import("../../stores/params").SensorParam} sensor
    * @param {import("../../stores/params").RegionParam} region
    */
-  function genSpec(stats, sensor, region) {
+  function genSpec(sensor, region) {
     const options = {
-      domain: sensor.domain(stats, region.level === 'state' || region.level === 'county' ? 'county' : 'state'),
+      domain: sensor.domain(region.level === 'state' || region.level === 'county' ? 'county' : 'state'),
       withStates: true,
       scheme: sensor.value.vegaColorScale,
     };
@@ -74,7 +74,7 @@
     return `${sensor.name}_${regionName}_${formatDateISO(date.value)}`;
   }
 
-  $: spec = genSpec($stats, sensor, region);
+  $: spec = genSpec(sensor, region);
   $: data = loadData(sensor, date);
 
   $: showsUS = region.level === 'nation';