Skip to content

Commit 947e023

Browse files
graftgraft
committed
update ingestion ui
1 parent 7ae0a78 commit 947e023

File tree

4 files changed

+98
-63
lines changed

4 files changed

+98
-63
lines changed

‎etna/packages/etna-js/components/metis_exploration.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ export function PickBucket({ project_name=CONFIG.project_name, setBucket, bucket
7171
return (
7272
<Grid container direction='row' className={className}>
7373
<Grid item container alignItems='flex-end' style={{width: 'auto'}}>
74-
<i className="fa fa-trash" aria-hidden="true" style={{padding:'3px 5px 10px 3px'}}/>
74+
<i className="fa fa-trash" aria-hidden="true" style={{padding:'5px 5px 5px 3px'}}/>
7575
</Grid>
7676
<Grid item style={{flex: '1 1 auto'}}>
7777
<Autocomplete

‎polyphemus/lib/client/jsx/workflow/ingestion-form.tsx

+52-28
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,26 @@ import React, {
66
useMemo
77
} from 'react';
88
import Grid from '@material-ui/core/Grid';
9-
import Tabs from '@material-ui/core/Tabs';
10-
import Tab from '@material-ui/core/Tab';
11-
import Typography from '@material-ui/core/Typography';
12-
import Tooltip from '@material-ui/core/Tooltip';
13-
import Select from '@material-ui/core/Select';
14-
import MenuItem from '@material-ui/core/MenuItem';
15-
import IconButton from '@material-ui/core/IconButton';
16-
import Button from '@material-ui/core/Button';
17-
import AddIcon from '@material-ui/icons/Add';
18-
import DeleteIcon from '@material-ui/icons/Delete';
19-
import CopyIcon from '@material-ui/icons/FileCopy';
20-
import Pagination from '@material-ui/lab/Pagination';
21-
import Switch from '@material-ui/core/Switch';
22-
import FormControlLabel from '@material-ui/core/FormControlLabel';
9+
import TextField from '@material-ui/core/TextField';
2310

2411
import {makeStyles, Theme} from '@material-ui/core/styles';
2512
import {SchemaContext, SchemaProvider} from './schema-context';
2613

14+
import {PickBucket} from 'etna-js/components/metis_exploration';
2715
import {Script, Job} from '../polyphemus';
2816
import AddModel from './add-model';
2917

3018
export const useStyles = makeStyles((theme: Theme) => ({
3119
}));
3220

21+
export type Config = {
22+
bucket_name: string,
23+
magic_string: string,
24+
deposit_root_path: string,
25+
metis_root_path: string,
26+
ingest_root_path: string
27+
};
28+
3329
const IngestionForm = ({
3430
config,
3531
project_name,
@@ -51,6 +47,8 @@ const IngestionForm = ({
5147

5248
const bucket_name = config.bucket_name || '';
5349

50+
const { magic_string, metis_root_path, ingest_root_path, deposit_root_path } = config;
51+
5452
const setBucket = useCallback(
5553
(bucket_name: string) => {
5654
let c = {...config, bucket_name };
@@ -62,28 +60,54 @@ const IngestionForm = ({
6260
return (
6361
<Grid container className={classes.form} direction='column'>
6462
<Grid item container>
65-
regex
66-
</Grid>
67-
<Grid item container>
68-
root_dir
69-
</Grid>
70-
<Grid item container>
71-
file_regex
72-
</Grid>
73-
<Grid item container>
74-
sftp_root_dir
63+
<Grid item xs={3} style={{paddingLeft:'10px'}}>Magic string</Grid>
64+
<Grid item xs={9}>
65+
<TextField
66+
fullWidth
67+
placeholder='Match string for target files'
68+
onChange={(e) => update({ ...config, magic_string: e.target.value})}
69+
value={magic_string}
70+
/>
71+
</Grid>
7572
</Grid>
7673
<Grid item container>
77-
path_to_write_files
74+
<Grid item xs={3} style={{paddingLeft:'10px'}}>Ingest root path</Grid>
75+
<Grid item xs={9}>
76+
<TextField
77+
fullWidth
78+
placeholder='Search path for files on ingest host'
79+
onChange={(e) => update({ ...config, ingest_root_path: e.target.value})}
80+
value={ingest_root_path}
81+
/>
82+
</Grid>
7883
</Grid>
7984
<Grid item container>
80-
bucket_name
85+
<Grid item xs={3} style={{paddingLeft:'10px'}}>Deposit root path</Grid>
86+
<Grid item xs={9}>
87+
<TextField
88+
fullWidth
89+
placeholder='Path to copy files onto deposit host'
90+
onChange={(e) => update({ ...config, deposit_root_path: e.target.value})}
91+
value={deposit_root_path}
92+
/>
93+
</Grid>
8194
</Grid>
8295
<Grid item container>
83-
metis_root_path
96+
<Grid item xs={3} style={{paddingLeft:'10px'}}>Metis bucket</Grid>
97+
<Grid item xs={9}>
98+
<PickBucket setBucket={setBucket} project_name={project_name} bucket={bucket_name} label={null}/>
99+
</Grid>
84100
</Grid>
85101
<Grid item container>
86-
deposit_root_path
102+
<Grid item xs={3} style={{paddingLeft:'10px'}}>Metis root path</Grid>
103+
<Grid item xs={9}>
104+
<TextField
105+
fullWidth
106+
placeholder='Path to copy files onto Metis'
107+
onChange={(e) => update({ ...config, metis_root_path: e.target.value})}
108+
value={metis_root_path}
109+
/>
110+
</Grid>
87111
</Grid>
88112
</Grid>
89113
);

‎polyphemus/lib/client/jsx/workflow/run-pane.tsx

+37-32
Original file line numberDiff line numberDiff line change
@@ -104,53 +104,58 @@ const Param = ({
104104
return (
105105
<DefaultSelect value={value} opts={opts} name={name} update={update} />
106106
);
107-
} else if (_.isObject(opts)) {
108-
if (opts.type && opts.type === 'options' && opts.value === 'model_names') {
109-
const modelNames = ['all'].concat(Object.keys(config).sort());
110-
return (
111-
<Autocomplete
112-
multiple
113-
fullWidth
114-
id={name}
115-
options={modelNames}
116-
value={'' === value ? [] : (value as string)?.split(',') || []}
117-
renderInput={(params) => <TextField {...params} variant='standard' />}
118-
onChange={(e, v) => {
119-
if (v.includes('all')) {
120-
update(name, 'all');
121-
} else {
122-
update(name, v.join(','));
123-
}
124-
}}
125-
/>
126-
);
127-
} else {
128-
return (
129-
<TextField
130-
size='small'
131-
fullWidth
132-
value={value == undefined ? '' : value}
133-
onChange={(e) => update(name, e.target.value)}
134-
/>
135-
);
136-
}
137-
} else if (opts == 'string') {
107+
}
108+
109+
if (!_.isObject(opts)) opts = { type: opts };
110+
111+
if (opts.type === 'options' && opts.value === 'model_names') {
112+
const modelNames = ['all'].concat(Object.keys(config).sort());
113+
return (
114+
<Autocomplete
115+
multiple
116+
fullWidth
117+
id={name}
118+
options={modelNames}
119+
value={'' === value ? [] : (value as string)?.split(',') || []}
120+
renderInput={(params) => <TextField {...params} variant='standard' />}
121+
onChange={(e, v) => {
122+
if (v.includes('all')) {
123+
update(name, 'all');
124+
} else {
125+
update(name, v.join(','));
126+
}
127+
}}
128+
/>
129+
);
130+
} else if (opts.type == 'string') {
138131
return (
139132
<TextField
140133
size='small'
134+
placeholder={opts.description}
141135
fullWidth
142136
value={value == undefined ? '' : value}
143137
onChange={(e) => update(name, e.target.value)}
144138
/>
145139
);
146-
} else if (opts == 'boolean') {
140+
} else if (opts.type == 'boolean') {
147141
return (
148142
<Switch
149143
size='small'
150144
checked={value == undefined ? false : (value as boolean)}
151145
onChange={(e) => update(name, e.target.checked)}
152146
/>
153147
);
148+
} else if (opts.type == 'integer') {
149+
return (
150+
<TextField
151+
size='small'
152+
type='number'
153+
placeholder={opts.description}
154+
fullWidth
155+
value={value == undefined ? '' : value}
156+
onChange={(e) => update(name, e.target.value)}
157+
/>
158+
);
154159
}
155160

156161
return null;

‎polyphemus/lib/data_eng/workflow_manifests/ingestion.rb

+8-2
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,14 @@ def as_json
1818
}
1919
},
2020
runtime_params: {
21-
initial_start_scan_time: 'integer', # unix timestamp
22-
override_interval: 'integer',
21+
initial_start_scan_time: {
22+
type: 'integer', # unix timestamp
23+
description: 'Scan start time, defaults to last completion'
24+
},
25+
override_interval: {
26+
type: 'integer',
27+
description: 'Span in seconds, defaults to current time'
28+
},
2329
restart_scan: 'boolean'
2430
},
2531
secrets: [

0 commit comments

Comments
 (0)