[TECH] Les types de colonnes ne sont pas corrects sur le datamart Parcoursup de dev (PIX-16800).

 #11550

Author: pix-service-auto-merge

api/datamart/migrations/20250303102000_fix_column_types.js

@@ -0,0 +1,30 @@
+const FIRST_DATAMART_TABLE_NAME = 'data_export_parcoursup_certif_result';
+const SECOND_DATAMART_TABLE_NAME = 'data_export_parcoursup_certif_result_code_validation';
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+const up = async function (knex) {
+  await knex.schema.alterTable(FIRST_DATAMART_TABLE_NAME, function (table) {
+    table.text('area_name').alter();
+    table.integer('certification_courses_id').alter();
+    table.text('competence_code').alter();
+    table.text('competence_name').alter();
+  });
+
+  await knex.schema.alterTable(SECOND_DATAMART_TABLE_NAME, function (table) {
+    table.integer('certification_courses_id').alter();
+    table.string('national_student_id', 50).alter();
+    table.string('organization_uai', 50).alter();
+  });
+};
+
+/**
+ * @returns { Promise<void> }
+ */
+const down = async function () {
+  // there are no reasons to go back to previous state as the columns types were utterly wrong
+};
+
+export { down, up };

api/datamart/seeds/cases/ine-ok.js

@@ -1,8 +1,8 @@
 import { faker } from '@faker-js/faker';
 
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generateFirstName,
   generatePixScore,
@@ -12,6 +12,7 @@ import {
   nationalStudentIdGenerator,
 } from './tools.js';
 
+const generateCertifCourseId = certificationCourseIdGenerator({ startingFrom: 1100000 });
 const generateINE = nationalStudentIdGenerator({ ineSuffix: 'AA' });
 
 /**

api/datamart/seeds/cases/same-ine-different-person.js

@@ -1,8 +1,8 @@
 import { faker } from '@faker-js/faker';
 
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generateFirstName,
   generatePixScore,
@@ -13,6 +13,8 @@ import {
   orgaUAIGenerator,
 } from './tools.js';
 
+const generateCertifCourseIdStudentOne = certificationCourseIdGenerator({ startingFrom: 3100000 });
+const generateCertifCourseIdStudentTwo = certificationCourseIdGenerator({ startingFrom: 4100000 });
 const generateINE = nationalStudentIdGenerator({ ineSuffix: 'CC' });
 const generateOrgaUai = orgaUAIGenerator();
 
@@ -23,7 +25,7 @@ export default function () {
   const sameINE = generateINE();
 
   const studentOneBase = {
-    certification_courses_id: generateCertifCourseId(),
+    certification_courses_id: generateCertifCourseIdStudentOne(),
     organization_uai: generateOrgaUai(),
     national_student_id: sameINE,
     last_name: faker.person.lastName(),
@@ -35,7 +37,7 @@ export default function () {
   };
 
   const studentTwoBase = {
-    certification_courses_id: generateCertifCourseId(),
+    certification_courses_id: generateCertifCourseIdStudentTwo(),
     organization_uai: generateOrgaUai(),
     national_student_id: sameINE,
     last_name: faker.person.lastName(),

api/datamart/seeds/cases/same-person-different-birthdate.js

@@ -1,8 +1,8 @@
 import { faker } from '@faker-js/faker';
 
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generateFirstName,
   generatePixScore,
@@ -13,6 +13,8 @@ import {
   orgaUAIGenerator,
 } from './tools.js';
 
+const generateCertifCourseIdStudentOne = certificationCourseIdGenerator({ startingFrom: 5100000 });
+const generateCertifCourseIdStudentTwo = certificationCourseIdGenerator({ startingFrom: 6100000 });
 const generateINE = nationalStudentIdGenerator({ ineSuffix: 'BB' });
 const generateOrgaUai = orgaUAIGenerator();
 
@@ -28,7 +30,7 @@ export default function () {
   const sameLastName = faker.person.lastName();
 
   const studentOneBase = {
-    certification_courses_id: generateCertifCourseId(),
+    certification_courses_id: generateCertifCourseIdStudentOne(),
     organization_uai: sameUAI,
     national_student_id: sameINE,
     last_name: sameLastName,
@@ -40,7 +42,7 @@ export default function () {
   };
 
   const studentTwoBase = {
-    certification_courses_id: generateCertifCourseId(),
+    certification_courses_id: generateCertifCourseIdStudentTwo(),
     organization_uai: sameUAI,
     national_student_id: sameINE,
     last_name: sameLastName,

api/datamart/seeds/cases/special-names.js

@@ -1,12 +1,14 @@
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generatePixScore,
   generateStatus,
   getCertificationDate,
 } from './tools.js';
 
+const generateCertifCourseId = certificationCourseIdGenerator({ startingFrom: 8100000 });
+
 /**
  * A student that has multiple accents in its first name and last name
  */

api/datamart/seeds/cases/tools.js

@@ -107,8 +107,9 @@ const generateStatus = () => {
   ]);
 };
 
-const generateCertifCourseId = () => {
-  return Date.now() + faker.number.int(10);
+const certificationCourseIdGenerator = ({ startingFrom = 1000 }) => {
+  const certifCourseIdGenerator = incrementalGenerator(startingFrom);
+  return () => certifCourseIdGenerator.next().value;
 };
 
 const generateCompetenceLevel = () => {
@@ -120,8 +121,8 @@ const generatePixScore = () => {
 };
 
 export {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generateFirstName,
   generatePixScore,

api/datamart/seeds/cases/uai-ok.js

@@ -1,13 +1,14 @@
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generatePixScore,
   generateStatus,
   getCertificationDate,
   orgaUAIGenerator,
 } from './tools.js';
 
+const generateCertifCourseId = certificationCourseIdGenerator({ startingFrom: 2100000 });
 const generateOrgaUai = orgaUAIGenerator();
 
 /**

api/datamart/seeds/cases/verification-code-only.js

@@ -1,8 +1,8 @@
 import { faker } from '@faker-js/faker';
 
 import {
+  certificationCourseIdGenerator,
   COMPETENCES,
-  generateCertifCourseId,
   generateCompetenceLevel,
   generateFirstName,
   generatePixScore,
@@ -12,6 +12,7 @@ import {
   verificationCodeGenerator,
 } from './tools.js';
 
+const generateCertifCourseId = certificationCourseIdGenerator({ startingFrom: 1000000 });
 const generateVerificationCode = verificationCodeGenerator();
 
 /**

api/datamart/seeds/populate-certification-results.js

@@ -1,5 +1,6 @@
 import { faker } from '@faker-js/faker';
 
+import { logger } from '../../src/shared/infrastructure/utils/logger.js';
 import caseINEok from './cases/ine-ok.js';
 import caseSameINEDifferentPerson from './cases/same-ine-different-person.js';
 import caseSamePersonDifferentBirthdate from './cases/same-person-different-birthdate.js';
@@ -12,34 +13,53 @@ const NUMBER_OF_SEEDS = Number(process.env.DATAMART_NUMBER_OF_SEEDS) || 100;
 const insertScoDatamart = async (knex) => {
   const scoDatamart = 'data_export_parcoursup_certif_result';
 
-  // Case 1 : INE ok
-  await knex.batchInsert(scoDatamart, faker.helpers.multiple(caseINEok, { count: NUMBER_OF_SEEDS }).flat());
-  // Case 2 : UAI ok
-  await knex.batchInsert(scoDatamart, faker.helpers.multiple(caseUAIok, { count: NUMBER_OF_SEEDS }).flat());
-  // Case 3 : Same INE different persons
-  await knex.batchInsert(
-    scoDatamart,
-    faker.helpers.multiple(caseSameINEDifferentPerson, { count: NUMBER_OF_SEEDS }).flat(),
-  );
-  // Case 4 : Same person but different birthdate
-  await knex.batchInsert(
-    scoDatamart,
-    faker.helpers.multiple(caseSamePersonDifferentBirthdate, { count: NUMBER_OF_SEEDS }).flat(),
-  );
-  // Case 5 : Complicated names (accents, dashes)
-  await knex.batchInsert(scoDatamart, faker.helpers.multiple(caseSpecialNames, { count: 1 }).flat());
+  logger.info('Start Case 1 : INE ok');
+  await _chunkify({ numberOfSeeds: NUMBER_OF_SEEDS, knex, datamart: scoDatamart, generateFn: caseINEok });
+
+  logger.info('Start Case 2 : UAI ok');
+  await _chunkify({ numberOfSeeds: NUMBER_OF_SEEDS, knex, datamart: scoDatamart, generateFn: caseUAIok });
+
+  logger.info('Start Case 3 : Same INE different persons');
+  await _chunkify({
+    numberOfSeeds: NUMBER_OF_SEEDS,
+    knex,
+    datamart: scoDatamart,
+    generateFn: caseSameINEDifferentPerson,
+  });
+
+  logger.info('Start Case 4 : Same person but different birthdate');
+  await _chunkify({
+    numberOfSeeds: NUMBER_OF_SEEDS,
+    knex,
+    datamart: scoDatamart,
+    generateFn: caseSamePersonDifferentBirthdate,
+  });
+
+  logger.info('Start Case 5 : Complicated names (accents, dashes');
+  await _chunkify({ numberOfSeeds: NUMBER_OF_SEEDS, knex, datamart: scoDatamart, generateFn: caseSpecialNames });
 };
 
 const insertGeneralPublicDatamart = async (knex) => {
+  logger.info('Start Case 6 : Verification code OK');
   const generalPublicDatamart = 'data_export_parcoursup_certif_result_code_validation';
-  await knex(generalPublicDatamart).truncate();
-  await knex.batchInsert(
-    generalPublicDatamart,
-    faker.helpers.multiple(caseVerificationCodeOK, { count: NUMBER_OF_SEEDS }).flat(),
-  );
+  await _chunkify({
+    numberOfSeeds: NUMBER_OF_SEEDS,
+    knex,
+    datamart: generalPublicDatamart,
+    generateFn: caseVerificationCodeOK,
+  });
 };
 
 export async function seed(knex) {
   await insertScoDatamart(knex);
   await insertGeneralPublicDatamart(knex);
 }
+
+const _chunkify = async ({ numberOfSeeds, knex, datamart, generateFn }) => {
+  const chunkSize = 100;
+  let remaining = numberOfSeeds;
+  do {
+    await knex.batchInsert(datamart, faker.helpers.multiple(generateFn, { count: chunkSize }).flat());
+    remaining = remaining - chunkSize;
+  } while (remaining / chunkSize >= 1);
+};