Merge pull request #228 from lidofinance/develop

Develop to main

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+<!--- If any section below doesn't make sense for your pull request, delete it please. -->
+
+### Description
+
+<!--- Briefly note most valuable changes in what you did and why we need it, even if the task was described in detail in the task tracker. -->
+
+### Demo
+
+<!--- If thee are visual changes, attach a link to a specific section on preview stand / add screenshots / record a [loom](https://www.loom.com/). -->
+
+### Code review notes
+
+<!--- Describe all uncertain decisions you made code-wise, e.g. readability vs performance. -->
+
+### Testing notes
+
+<!--- List all possible edge cases and how to test them. -->
+
+### Checklist:
+
+- [ ] Checked the changes locally.
+- [ ] Created / updated analytics events.
+- [ ] Created / updated the technical documentation (README.md / [docs](https://docs.lido.fi/) / etc.).
+- [ ] Affects / requires changes in other services (Matomo / Sentry / CloudFlare / etc.).

.github/workflows/ci-dev.yml

@@ -31,12 +31,3 @@ jobs:
           TARGET_REPO: "lidofinance/infra-mainnet"
           TARGET_WORKFLOW: "deploy_holesky_testnet_withdrawals_api.yaml"
           TARGET: "develop"
-
-      - name: Goerli testnet deploy
-        uses: lidofinance/dispatch-workflow@v1
-        env:
-          APP_ID: ${{ secrets.APP_ID }}
-          APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
-          TARGET_REPO: "lidofinance/infra-mainnet"
-          TARGET_WORKFLOW: "deploy_testnet_withdrawals_api.yaml"
-          TARGET: "develop"

.github/workflows/tests.yml

@@ -22,12 +22,11 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: "16"
+          node-version: "20"
 
       - name: Install dependencies
         run: yarn install --frozen-lockfile
 
       - name: Run tests
         run: yarn test
 
-

.gitignore

@@ -35,4 +35,7 @@ lerna-debug.log*
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
-!.vscode/extensions.json
+!.vscode/extensions.json
+
+# cache
+cache

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+## Contributing Guidelines
+
+Thank you for your interest in contributing to the project! Your enthusiasm and support are appreciated. Comprehensive contributing guidelines are actively being developed to ensure a smooth and collaborative contribution process.
+
+**Note:** At this moment, there is no guarantee that pull requests or issues will be processed, as the workflow and guidelines are still being established to make participation easier for everyone in the project. The importance of community involvement is understood, and an inclusive and comfortable environment for all contributors is being created.
+
+Rest assured, as soon as the contributing guide is ready, contributions and any encountered issues will be welcomed and addressed. An announcement will be made when the guidelines are finalized and open for contributions.
+
+In the meantime, exploring the project, familiarizing yourself with the codebase, and providing valuable feedback or suggestions during this development phase are encouraged.
+
+Thank you for your patience and understanding. Future contributions are eagerly anticipated once the contributing guidelines are in place!

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18-alpine as building
+FROM node:20-alpine as building
 
 WORKDIR /app
 
@@ -9,7 +9,7 @@ COPY ./src ./src
 RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean
 RUN yarn build
 
-FROM node:18-alpine
+FROM node:20-alpine
 
 WORKDIR /app
 

README.md

@@ -17,7 +17,7 @@ The service is helpful for stakers, providing insights from the moment of withdr
 
 ### Prerequisites
 
-- Node.js (version 16.0 or higher)
+- Node.js (version 20.0 or higher)
 - Yarn (version 1.22 or higher)
 
 ### Installation
@@ -46,6 +46,7 @@ $ yarn start
 $ yarn start:dev
 
 # production mode
+$ yarn build
 $ yarn start:prod
 ```
 

how-estimation-works.md

@@ -91,21 +91,26 @@ It's needed to select the Lido-participating validators which are already in pro
 - projectedRewards (`rewardsPerEpoch * epochsTillTheFrame`)
 - frameBalances (`object { [frame]: [sum of balances of validators with withdrawable_epoch for certain frame] }`)
 
+So the final formula for that case looks like this:
+`frame (which has engough validator balances) + sweepingMean`. More about `sweepingMean` [here](#sweeping mean).
+
 ---
 
 ### 3.iii. Case when new validator exits are needed to finalize the withdrawal requests
 
 - The idea is to find the nearest epoch pretending that all needed validators were exited
 - Find epoch by this formula: `unfinalizedStETH / (32ETH * churnLimit + rewardsPerEpoch)`
 - `rewardsPerEpoch` is calculated as described in the provided [prediction model](https://hackmd.io/@lido/r1fau3aJ3?type=view#Predict-available-ETH-before-next-withdrawn)
+- Worth noting that the exited validators become withdrawable only after the withdrawal  [sweep](#sweeping mean) approached them.
 
-- Worth noting that the exited validators become withdrawable only after the withdrawal sweep approached them.
+### Sweeping Mean
 For simplicity, it's suggested to use the average time of the withdrawal sweep (`sweepingMean`) as a constant timeframe extension when estimating the withdrawal waiting time.
 
 More information can be found [here](https://consensys.net/shanghai-capella-upgrade/) in *Full Withdrawal Process* chapter*.*
 
 > it is takes 256 epoch (27.3 hours) at minimum on Mainnet, and the current avg value is 567 epochs (the values depend on the total validators number).
 
+
 ---
 
 ### Conclusion

package.json

@@ -6,7 +6,7 @@
   "private": true,
   "license": "MIT",
   "engines": {
-    "node": ">=16",
+    "node": ">=20",
     "yarn": ">=1.22"
   },
   "scripts": {

src/common/config/env.validation.ts

@@ -58,7 +58,7 @@ export class EnvironmentVariables {
 
   @IsOptional()
   @IsString()
-  JOB_INTERVAL_VALIDATORS = CronExpression.EVERY_3_HOURS;
+  JOB_INTERVAL_VALIDATORS;
 
   @IsOptional()
   @IsString()

src/http/request-time/dto/request-time.dto.ts

@@ -1,33 +1,41 @@
 import { ApiProperty } from '@nestjs/swagger';
+import { WaitingTimeStatus } from '../../../waiting-time';
 
 export class RequestTimeDto {
   @ApiProperty({
     example: 5,
     description: 'Maximum waiting days',
   })
-  days: number;
+  days?: number;
 
   @ApiProperty({
     example: 0,
     description: 'Queue ETH last update timestamp',
   })
-  stethLastUpdate: number;
+  stethLastUpdate?: number;
 
   @ApiProperty({
     example: 0,
     description: 'Validators last update timestamp',
   })
-  validatorsLastUpdate: number;
+  validatorsLastUpdate?: number;
 
   @ApiProperty({
     example: 10,
     description: 'Queue requests count',
   })
-  requests: number;
+  requests?: number;
 
   @ApiProperty({
     example: 0,
     description: 'Queue steth amount',
   })
-  steth: string;
+  steth?: string;
+
+  @ApiProperty({
+    type: 'string',
+    description: 'Status of request calculation',
+    enum: Object.values(WaitingTimeStatus),
+  })
+  status: WaitingTimeStatus;
 }

src/http/request-time/request-time.service.ts

@@ -5,7 +5,7 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
 import { parseEther } from '@ethersproject/units';
 
 import { QueueInfoStorageService, ValidatorsStorageService } from 'storage';
-import { WaitingTimeService } from 'waiting-time';
+import { WaitingTimeService, WaitingTimeStatus } from 'waiting-time';
 
 import { RequestTimeDto, RequestTimeOptionsDto } from './dto';
 import { RequestTimeV2Dto } from './dto/request-time-v2.dto';
@@ -23,10 +23,18 @@ export class RequestTimeService {
 
   async getRequestTime(params: RequestTimeOptionsDto): Promise<RequestTimeDto | null> {
     const validatorsLastUpdate = this.validators.getLastUpdate();
-    if (!validatorsLastUpdate) return null;
+    if (!validatorsLastUpdate) {
+      return {
+        status: WaitingTimeStatus.initializing,
+      };
+    }
 
     const unfinalizedETH = this.queueInfo.getStETH();
-    if (!unfinalizedETH) return null;
+    if (!unfinalizedETH) {
+      return {
+        status: WaitingTimeStatus.initializing,
+      };
+    }
 
     const additionalStETH = parseEther(params.amount || '0');
     const queueStETH = unfinalizedETH.add(additionalStETH);
@@ -47,6 +55,7 @@ export class RequestTimeService {
       validatorsLastUpdate,
       steth: unfinalizedETH.toString(),
       requests: requestsCount.toNumber(),
+      status: WaitingTimeStatus.calculated,
     };
   }
 

src/jobs/validators/validators.constants.ts

@@ -1,4 +1,11 @@
 import { BigNumber } from '@ethersproject/bignumber';
+import { CHAINS } from '@lido-nestjs/constants';
+import { CronExpression } from '@nestjs/schedule';
 
 export const FAR_FUTURE_EPOCH = BigNumber.from(2).pow(64).sub(1);
 export const MAX_SEED_LOOKAHEAD = 4;
+
+export const ORACLE_REPORTS_CRON_BY_CHAIN_ID = {
+  [CHAINS.Mainnet]: '30 4/8 * * *', // 4 utc, 12 utc, 20 utc
+  [CHAINS.Holesky]: CronExpression.EVERY_3_HOURS, // happens very often, not necessary sync in testnet
+};

src/jobs/validators/validators.service.ts

@@ -7,13 +7,15 @@ import { ConsensusProviderService } from 'common/consensus-provider';
 import { GenesisTimeService } from 'common/genesis-time';
 import { OneAtTime } from '@lido-nestjs/decorators';
 import { ValidatorsStorageService } from 'storage';
-import { FAR_FUTURE_EPOCH, MAX_SEED_LOOKAHEAD } from './validators.constants';
+import { FAR_FUTURE_EPOCH, ORACLE_REPORTS_CRON_BY_CHAIN_ID, MAX_SEED_LOOKAHEAD } from './validators.constants';
 import { BigNumber } from '@ethersproject/bignumber';
 import { processValidatorsStream } from 'jobs/validators/utils/validators-stream';
 import { unblock } from '../../common/utils/unblock';
 import { LidoKeysService } from './lido-keys';
 import { ResponseValidatorsData, Validator } from './validators.types';
 import { parseGweiToWei } from '../../common/utils/parse-gwei-to-big-number';
+import { ValidatorsCacheService } from 'storage/validators/validators-cache.service';
+import { CronExpression } from '@nestjs/schedule';
 
 export class ValidatorsService {
   constructor(
@@ -23,6 +25,7 @@ export class ValidatorsService {
     protected readonly configService: ConfigService,
     protected readonly jobService: JobService,
     protected readonly validatorsStorageService: ValidatorsStorageService,
+    protected readonly validatorsCacheService: ValidatorsCacheService,
     protected readonly genesisTimeService: GenesisTimeService,
     protected readonly lidoKeys: LidoKeysService,
   ) {}
@@ -31,9 +34,13 @@ export class ValidatorsService {
    * Initializes the job
    */
   public async initialize(): Promise<void> {
+    await this.validatorsCacheService.initializeFromCache();
     await this.updateValidators();
 
-    const cronTime = this.configService.get('JOB_INTERVAL_VALIDATORS');
+    const envCronTime = this.configService.get('JOB_INTERVAL_VALIDATORS');
+    const chainId = this.configService.get('CHAIN_ID');
+    const cronByChainId = ORACLE_REPORTS_CRON_BY_CHAIN_ID[chainId] ?? CronExpression.EVERY_3_HOURS;
+    const cronTime = envCronTime ? envCronTime : cronByChainId;
     const job = new CronJob(cronTime, () => this.updateValidators());
     job.start();
 
@@ -72,6 +79,8 @@ export class ValidatorsService {
       this.validatorsStorageService.setMaxExitEpoch(latestEpoch);
       this.validatorsStorageService.setLastUpdate(Math.floor(Date.now() / 1000));
 
+      await this.validatorsCacheService.saveDataToCache();
+
       this.logger.log('End update validators', { service: 'validators', totalValidators, latestEpoch });
     });
   }

src/storage/validators/index.ts

@@ -1,2 +1,3 @@
 export * from './validators.module';
 export * from './validators.service';
+export * from './validators-cache.service';