Skip to content

[ui-importer] finish import API integration #4122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ import {
GuessFieldTypesResponse,
ImporterTableData
} from '../types';
import { convertToAntdColumns, convertToDataSource } from '../utils/utils';
import { convertToAntdColumns, convertToDataSource, getDefaultTableName } from '../utils/utils';
import { i18nReact } from '../../../utils/i18nReact';
import { BorderlessButton, PrimaryButton } from 'cuix/dist/components/Button';
import PaginatedTable from '../../../reactComponents/PaginatedTable/PaginatedTable';
import { GUESS_FORMAT_URL, GUESS_FIELD_TYPES_URL } from '../api';
import { GUESS_FORMAT_URL, GUESS_FIELD_TYPES_URL, FINISH_IMPORT_URL } from '../api';
import SourceConfiguration from './SourceConfiguration/SourceConfiguration';

import './ImporterFilePreview.scss';
Expand All @@ -39,6 +39,8 @@ const ImporterFilePreview = ({ fileMetaData }: ImporterFilePreviewProps): JSX.El
const { t } = i18nReact.useTranslation();
const [fileFormat, setFileFormat] = useState<FileFormatResponse | undefined>();

const defaultTableName = getDefaultTableName(fileMetaData.path, fileMetaData.source);

const { save: guessFormat, loading: guessingFormat } = useSaveData<FileFormatResponse>(
GUESS_FORMAT_URL,
{
Expand All @@ -54,6 +56,9 @@ const ImporterFilePreview = ({ fileMetaData }: ImporterFilePreviewProps): JSX.El
loading: guessingFields
} = useSaveData<GuessFieldTypesResponse>(GUESS_FIELD_TYPES_URL);

const { save, loading: finalizingImport } =
useSaveData<GuessFieldTypesResponse>(FINISH_IMPORT_URL);

useEffect(() => {
const guessFormatPayload = {
inputFormat: fileMetaData.source,
Expand All @@ -80,6 +85,32 @@ const ImporterFilePreview = ({ fileMetaData }: ImporterFilePreviewProps): JSX.El
guessFields(formData);
}, [fileMetaData.path, fileFormat]);

const handleFinishImport = () => {
// TODO: take the hardcoded values from the form once implemented
const dialect = 'impala';
const database = 'default';

const source = {
inputFormat: fileMetaData.source,
path: fileMetaData.path,
format: fileFormat,
sourceType: dialect
};
const destination = {
outputFormat: 'table',
nonDefaultLocation: fileMetaData.path,
name: `${database}.${defaultTableName}`,
sourceType: dialect,
columns: previewData?.columns
};

const formData = new FormData();
formData.append('source', JSON.stringify(source));
formData.append('destination', JSON.stringify(destination));

save(formData);
};

const columns = convertToAntdColumns(previewData?.columns ?? []);
const tableData = convertToDataSource(columns, previewData?.sample);

Expand All @@ -91,7 +122,7 @@ const ImporterFilePreview = ({ fileMetaData }: ImporterFilePreviewProps): JSX.El
<BorderlessButton data-testid="hue-importer-preview-page__header__actions__cancel">
{t('Cancel')}
</BorderlessButton>
<PrimaryButton data-testid="hue-importer-preview-page__header__actions__finish">
<PrimaryButton loading={finalizingImport} onClick={handleFinishImport}>
{t('Finish Import')}
</PrimaryButton>
</div>
Expand Down
1 change: 1 addition & 0 deletions desktop/core/src/desktop/js/apps/newimporter/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@
export const UPLOAD_LOCAL_FILE_API_URL = '/indexer/api/indexer/upload_local_file';
export const GUESS_FORMAT_URL = '/indexer/api/indexer/guess_format';
export const GUESS_FIELD_TYPES_URL = '/indexer/api/indexer/guess_field_types';
export const FINISH_IMPORT_URL = '/indexer/api/importer/submit';
50 changes: 47 additions & 3 deletions desktop/core/src/desktop/js/apps/newimporter/utils/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// limitations under the License.

import { ColumnProps } from 'cuix/dist/components/Table';
import { convertToAntdColumns, convertToDataSource } from './utils';
import { GuessFieldTypesColumn } from '../types';
import { convertToAntdColumns, convertToDataSource, getDefaultTableName } from './utils';
import { ImporterFileSource, GuessFieldTypesColumn, ImporterTableData } from '../types';

describe('convertToAntdColumns', () => {
it('should return an empty array when no input is provided', () => {
Expand All @@ -35,7 +35,7 @@ describe('convertToAntdColumns', () => {
});

describe('convertToDataSource', () => {
const columns: ColumnProps<GuessFieldTypesColumn>[] = [
const columns: ColumnProps<ImporterTableData>[] = [
{ title: 'Name', dataIndex: 'name', key: 'name', width: '100px' },
{ title: 'Age', dataIndex: 'age', key: 'age', width: '100px' }
];
Expand All @@ -58,3 +58,47 @@ describe('convertToDataSource', () => {
expect(convertToDataSource(columns, apiResponse)).toEqual(expectedOutput);
});
});

describe('getDefaultTableName', () => {
it('should extract file name from LOCALFILE path using pattern', () => {
const filePath = '/user/data/:myDocument;v1.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.LOCAL);
expect(result).toBe('myDocument');
});

it('should return empty string if LOCALFILE pattern does not match', () => {
const filePath = '/user/data/myDocument.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.LOCAL);
expect(result).toBe('');
});

it('should extract file name from REMOTE path as last part of path', () => {
const filePath = 'https://demo.gethue.com/hue/test-file.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.REMOTE);
expect(result).toBe('test-file');
});

it('should handle file names with multiple dots correctly', () => {
const filePath = 'https://demo.gethue.com/hue/test.file.name.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.REMOTE);
expect(result).toBe('test_file_name');
});

it('should handle file names with no extension correctly', () => {
const filePath = 'https://demo.gethue.com/hue/test-file';
const result = getDefaultTableName(filePath, ImporterFileSource.REMOTE);
expect(result).toBe('test-file');
});

it('should handle file names with special characters correctly', () => {
const filePath = 'https://demo.gethue.com/hue/test-file@2023.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.REMOTE);
expect(result).toBe('test-file@2023');
});

it('should handle file names with spaces correctly', () => {
const filePath = 'https://demo.gethue.com/hue/test file.csv';
const result = getDefaultTableName(filePath, ImporterFileSource.REMOTE);
expect(result).toBe('test file');
});
});
21 changes: 20 additions & 1 deletion desktop/core/src/desktop/js/apps/newimporter/utils/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
// limitations under the License.

import { type ColumnProps } from 'cuix/dist/components/Table';
import { GuessFieldTypesColumn, ImporterTableData } from '../types';
import { ImporterFileSource, GuessFieldTypesColumn, ImporterTableData } from '../types';
import { getLastDirOrFileNameFromPath } from '../../../reactComponents/PathBrowser/PathBrowser.util';

export const convertToAntdColumns = (
input?: GuessFieldTypesColumn[]
Expand Down Expand Up @@ -50,3 +51,21 @@ export const convertToDataSource = (
return row;
});
};

export const getDefaultTableName = (filePath: string, fileSource: ImporterFileSource): string => {
// For local files, the file name is extracted from the path
// Example: /**/**/**:fileName;**.fileExtension
if (fileSource === ImporterFileSource.LOCAL) {
const match = filePath.match(/:(.*?);/);
return match?.[1] ?? '';
}

// For Remote, remove extension and replace '.' with '_'
// Example: file.name.fileExtension -> file_name
const fileName = getLastDirOrFileNameFromPath(filePath);
if (fileName.split('.').length === 1) {
// If there is no extension, return the file name as is
return fileName;
}
return fileName.split('.').slice(0, -1).join('_');
};