Merge branch 'main' into if-merge

Author: manushak

.github/workflows/dispatch-integration-test.yml

@@ -0,0 +1,23 @@
+name: Run if-check on requested branch
+
+on: 
+  workflow_dispatch: 
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run integration tests
+        run: npm run if-check -- -d manifests/outputs

.github/workflows/release-commit.yml

@@ -8,7 +8,7 @@ env:
   PRE_RELEASE: ${{ github.event.release.prerelease}}
 
 jobs:
-  release-commit-pr:
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -28,9 +28,45 @@ jobs:
       - name: Run unit tests
         run: npm test
 
+  integration-tests:
+    runs-on: ubuntu-latest
+    needs: unit-tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+
+      - name: Install dependencies
+        run: npm ci
+
       - name: Run integration tests
         run: npm run if-check -- -d manifests/outputs
 
+  publish:
+    runs-on: ubuntu-latest
+    needs: integration-tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+
+      - name: Install dependencies
+        run: npm ci
+
       - name: Initialize git user email
         run: git config --global user.email "${{ vars.RELEASE_USER_EMAIL }}"
 

Refactor-migration-guide.md

@@ -125,24 +125,23 @@ There have also been some changes to the structure of manifest files. Some of th
 
 - **Node level config**
 
-  We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `groupby` plugin to group the outputs according to a different set of keys.
+  We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys.
 
   ```yaml
   tree:
     children:
       child-1:
         pipeline:
-          - teads-curve
-          - sci-e
-          - sci-embodied
-          - sci-o
-          - time-sync
-          - sci
-        config:
-          group-by:
-            group:
-              - region
-              - cloud/instance-type
+          compute:
+            - teads-curve
+            - sci-e
+            - sci-embodied
+            - sci-o
+            - time-sync
+            - sci
+          regroup:
+            - region
+            - cloud/instance-type
   ```
 
 - **Defaults**
@@ -156,12 +155,13 @@ There have also been some changes to the structure of manifest files. Some of th
     children:
       child-1:
         pipeline:
-          - teads-curve
-          - sci-e
-          - sci-embodied
-          - sci-o
-          - time-sync
-          - sci
+          compute:
+            - teads-curve
+            - sci-e
+            - sci-embodied
+            - sci-o
+            - time-sync
+            - sci
        defaults:
           cpu/thermal-design-power: 100
           grid/carbon-intensity: 800
@@ -199,9 +199,7 @@ The aggregate plugin aggregates data in two ways: first it condenses individual
 
 This is a builtin feature of IF, meaning it does not have to be initialized as a plugin. Instead, you just have to include a short config block in the top of the manifest file. There are two pieces of information required:
 
-- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array.
-
-  - `method`: the aggregation method for the specied metric
+- `metrics`: which metrics do you want to aggregate? Every metric you provide here must exist in the output array and be described in the `parameter-metadata` of the plugin.
 
 - `type`: the options are `horizontal`, `vertical` or both. Horizontal aggregation is the type that condenses each time series into a single summary value. Vertical aggregation is aggregated across components.
 
@@ -210,45 +208,10 @@ Here's what the config block should look like:
 ```yaml
 aggregation:
   metrics:
-    'carbon':
-      method: 'sum'
+    - carbon
   type: 'both'
 ```
 
-### Groupby
-
-Groupby allows you to regroup your outputs according to keys you define. For example, maybe you want to group your outputs by region (show me all the outputs for applications run in `uk-south` etc). Groupby _is_ a plugin that needs to be initialized in the manifest.
-
-You can initialize the plugin as follows:
-
-```yaml
-initialize:
-  plugins:
-    'group-by':
-      path: builtin
-      method: GroupBy
-```
-
-Then you configure groupby for each component in the node level config. In the following example we will regroup the outputs by the `region`:
-
-```yaml
-tree:
-  children:
-    child-1:
-      pipeline:
-        - teads-curve
-        - sci-e
-        - sci-embodied
-        - sci-o
-        - time-sync
-        - group-by
-        - sci
-      config:
-        group-by:
-          group:
-            - region
-```
-
 ### Exhaust
 
 We have introduced `exhaust` as an IF feature. This is a wrapper around export plugins and it allows community contributors to create plugins for exporting to different formats.

manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml

@@ -14,7 +14,8 @@ tree:
   children:
     child:
       pipeline:
-        - coefficient
+        compute:
+          - coefficient
       config:
         sum:
       inputs:

manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml

@@ -14,7 +14,8 @@ tree:
   children:
     child:
       pipeline:
-        - coefficient
+        compute:
+          - coefficient
       config:
         sum:
       inputs:

manifests/examples/builtins/coefficient/success.yml

@@ -14,7 +14,8 @@ tree:
   children:
     child:
       pipeline:
-        - coefficient
+        compute:
+          - coefficient
       config:
         sum:
       inputs:

manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       config:
       inputs:
         - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred

manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       config:
       inputs:
         - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred

manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       config:
       inputs:
         - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred

manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       config:
       inputs:
         - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred

manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       inputs:
         - timestamp: 2023-08-06T00:00
           duration: 3600

manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       inputs:
         - timestamp: 2023-08-06T00:00
           duration: 3600

manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       inputs:
         - timestamp: 2023-08-06T00:00
           duration: 3600

manifests/examples/builtins/csv-lookup/region-metadata/success.yml

@@ -16,7 +16,8 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
+        compute:
+          - cloud-metadata
       inputs:
         - timestamp: 2023-08-06T00:00
           duration: 3600

manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml

@@ -15,7 +15,8 @@ tree:
   children:
     child:
       pipeline:
-        - tdp-finder
+        compute:
+          - tdp-finder
       config:
       inputs:
         - timestamp: 2023-07-06T00:00

manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml

@@ -15,7 +15,8 @@ tree:
   children:
     child:
       pipeline:
-        - tdp-finder
+        compute:
+          - tdp-finder
       config:
       inputs:
         - timestamp: 2023-07-06T00:00

manifests/examples/builtins/csv-lookup/tdp-finder/success.yml

@@ -15,7 +15,8 @@ tree:
   children:
     child:
       pipeline:
-        - tdp-finder
+        compute:
+          - tdp-finder
       config:
       inputs:
         - timestamp: 2023-07-06T00:00

manifests/examples/builtins/divide/failure-denominator-equal-zero.yml

@@ -23,8 +23,9 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
-        - divide
+        compute:
+          - cloud-metadata
+          - divide
       config:
         divide:
       defaults:

manifests/examples/builtins/divide/failure-invalid-config-denominator.yml

@@ -23,8 +23,9 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
-        - divide
+        compute:
+          - cloud-metadata
+          - divide
       config:
         divide:
       defaults:

manifests/examples/builtins/divide/failure-missing-numerator.yml

@@ -23,8 +23,9 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
-        - divide
+        compute:
+          - cloud-metadata
+          - divide
       config:
         divide:
       defaults:

manifests/examples/builtins/divide/success.yml

@@ -23,10 +23,9 @@ tree:
   children:
     child:
       pipeline:
-        - cloud-metadata
-        - divide
-      config:
-        divide:
+        compute:
+          - cloud-metadata
+          - divide
       defaults:
         cloud/vendor: aws
         cloud/instance-type: m5n.large

manifests/examples/builtins/exponent/success.yml

@@ -14,9 +14,8 @@ tree:
   children:
     child:
       pipeline:
-        - exponent
-      config:
-        exponent:
+        compute:
+          - exponent
       inputs:
         - timestamp: 2023-08-06T00:00
           duration: 3600