Skip to content

Commit c3d7855

Browse files
sbracalonisbracaloni
committed
DVC tutorial: structure
- docker-compose + Dockerfile: git repo, dvc repo + 2 users - First pipeline resources - New pipeline feature resources DVC tutorial: add playground step by step explanation - Create a pipeline - Add remote storage - Share - Reproduce pipeline
1 parent c5c24dc commit c3d7855

File tree

15 files changed

+914
-0
lines changed

15 files changed

+914
-0
lines changed

‎resources/dvc_playground/docker-compose.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
version: '3'
2+
services:
3+
remote_dvc_repo:
4+
build:
5+
context: ./remote_dvc
6+
ports:
7+
- "22:22"
8+
9+
remote_git_repo:
10+
build:
11+
context: ./remote_git
12+
13+
user1:
14+
build:
15+
context: ./user
16+
args:
17+
USER_NAME: "songoku"
18+
command: tail -f /dev/null
19+
links:
20+
- remote_dvc_repo:dvc_srv
21+
- remote_git_repo:git_srv
22+
user2:
23+
build:
24+
context: ./user
25+
args:
26+
USER_NAME: "bulma"
27+
command: tail -f /dev/null
28+
links:
29+
- remote_dvc_repo:dvc_srv
30+
- remote_git_repo:git_srv

‎resources/dvc_playground/remote_dvc/Dockerfile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM debian:stretch
2+
RUN apt-get update && \
3+
apt-get -y install openssh-server vim && \
4+
mkdir -p /var/run/sshd
5+
6+
RUN groupadd ssh_user
7+
RUN useradd -g ssh_user -d /upload -s /bin/bash poney -p azerty
8+
RUN mkdir -p /data/dvc/remote
9+
RUN chown -R root:ssh_user /data/dvc
10+
RUN chown -R poney:ssh_user /data/dvc/remote
11+
RUN chmod ug+w -R /data/dvc/remote
12+
13+
COPY pub_key /tmp
14+
RUN useradd -g ssh_user -m -d /home/dvc_user -s /bin/bash dvc_user && \
15+
mkdir -p /home/dvc_user/.ssh/ && \
16+
cat /tmp/pub_key > /home/dvc_user/.ssh/authorized_keys && \
17+
chown dvc_user:ssh_user -R /home/dvc_user && \
18+
chmod 644 /home/dvc_user/.ssh/authorized_keys
19+
20+
21+
CMD ["/usr/sbin/sshd", "-D"]

‎resources/dvc_playground/remote_dvc/pub_key

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LtCP20Y3Dxy1I7lVB98PqoUuFS7ggeC3pHC32r8e8Vfwhj73TVSHPvdUQDNLWXt+NuZpdSFrpWoT6l1YMHyRZArC1QiM0t+4ptt8Fr1baupnXHO3I74gp89+XEMdDjqSR9WZ0MIX1KRV956samUuEdHmEMTw22HaUQBCrk1b2P9J7e5AchNXJWMWITq9Rorzg58Pquj3ejENKlotAAyVzTAwrnfUuTlmTC96GXarJp4Pkx4LlWJv4J18XfrBRfGUD5F23IBMiII9fQtxrQZ3ntb3TALGCFmxs8udT0eXefsej10iAvrllP5Qg70fNKHpD31qn570AirbT+4FEq91 sbracaloni@poney

‎resources/dvc_playground/remote_git/Dockerfile

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM debian:stretch
2+
RUN apt-get update && \
3+
apt-get -y install openssh-server git vim && \
4+
mkdir -p /var/run/sshd
5+
6+
7+
RUN useradd -d /home/git -m -s /bin/bash git
8+
9+
10+
RUN mkdir /home/git/.ssh && chmod 700 /home/git/.ssh
11+
COPY pub_key /home/git/.ssh/authorized_keys
12+
RUN chmod 600 /home/git/.ssh/authorized_keys
13+
RUN chown git: -R /home/git/.ssh
14+
RUN mkdir -p /srv/git/test_dvc_remote.git
15+
RUN cd /srv/git/test_dvc_remote.git/ && git init --bare
16+
RUN chown git: /srv/git/ -R
17+
18+
19+
CMD ["/usr/sbin/sshd", "-D"]

‎resources/dvc_playground/remote_git/pub_key

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LtCP20Y3Dxy1I7lVB98PqoUuFS7ggeC3pHC32r8e8Vfwhj73TVSHPvdUQDNLWXt+NuZpdSFrpWoT6l1YMHyRZArC1QiM0t+4ptt8Fr1baupnXHO3I74gp89+XEMdDjqSR9WZ0MIX1KRV956samUuEdHmEMTw22HaUQBCrk1b2P9J7e5AchNXJWMWITq9Rorzg58Pquj3ejENKlotAAyVzTAwrnfUuTlmTC96GXarJp4Pkx4LlWJv4J18XfrBRfGUD5F23IBMiII9fQtxrQZ3ntb3TALGCFmxs8udT0eXefsej10iAvrllP5Qg70fNKHpD31qn570AirbT+4FEq91

‎resources/dvc_playground/user/Dockerfile

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM python:3.6
2+
ARG USER_NAME
3+
RUN apt-get update && \
4+
apt-get -y install openssh-client vim git tree && \
5+
mkdir -p /var/run/sshd
6+
7+
COPY private_key /tmp/
8+
9+
RUN useradd -d /home/$USER_NAME -m -s /bin/bash $USER_NAME
10+
11+
RUN mkdir /home/$USER_NAME/.ssh
12+
COPY private_key /home/$USER_NAME/.ssh/id_rsa
13+
RUN chown $USER_NAME:$USER_NAME -R /home/$USER_NAME/
14+
RUN chmod 600 /home/$USER_NAME/.ssh/id_rsa
15+
16+
USER $USER_NAME
17+
ENV PATH=$PATH:/home/$USER_NAME/.local/bin/
18+
RUN pip install --user dvc paramiko
19+
20+
RUN git config --global user.name $(whoami)
21+
RUN git config --global user.email $(whoami)@example.com
22+
COPY resources /resources
23+
24+
EXPOSE 22

‎resources/dvc_playground/user/dvc_init_repo.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env bash
2+
3+
pushd ~
4+
git config --global user.name $(whoami)
5+
git config --global user.email $(whoami)@example.com
6+
7+
git clone git@git_srv:/srv/git/test_dvc_remote.git
8+
popd
9+
10+
11+
dvc remote add dvc_remote ssh://dvc_user@dvc_srv:/data/dvc/remote
12+
dvc config core.remote dvc_remote
13+
tail -f /dev/null

‎resources/dvc_playground/user/private_key

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEowIBAAKCAQEAti7Qj9tGNw8ctSO5VQffD6qFLhUu4IHgt6Rwt9q/HvFX8IY+
3+
901Uhz73VEAzS1l7fjbmaXUha6VqE+pdWDB8kWQKwtUIjNLfuKbbfBa9W2rqZ1xz
4+
tyO+IKfPflxDHQ46kkfVmdDCF9SkVfeerGplLhHR5hDE8Nth2lEAQq5NW9j/Se3u
5+
QHITVyVjFiE6vUaK84OfD6ro93oxDSpaLQAMlc0wMK531Lk5Zkwvehl2qyaeD5Me
6+
C5Vib+CdfF36wUXxlA+RdtyATIiCPX0Lca0Gd57W90wCxghZsbPLnU9Hl3n7Ho9d
7+
IgL65ZT+UIO9HzSh6Q99ap+e9AIq20/uBRKvdQIDAQABAoIBAGuQ8pWtqySBKLsO
8+
77Nz5VFTpjTSTw5BwRPM5Ly5TFURvJGIQg80Om9sTw4QpficSdz4v0yO2v+6+Anw
9+
O9MBNduGTRBCRj0XmP5mjnsS1hTu6J3EqZH9aZnQWyPJh9rU9uI9xbF+38rAQ5N6
10+
k9ZLL5hm7x6+0flAHuOPJe85CPXsBm8wRX7J7h9865uX+L7zvt7rRWBtGS5gqVj9
11+
uRm+wwJiOzjc86Q1h725Ev7pKchYMtFPNIMmrrSQsdRinhDyNCPamZI2YISWNaHM
12+
3814MbPi9M3v71YBFp80JAcaebgRO/enC4f/qUaaHtVtV65cgoaacLytdBf120n9
13+
MqRLT7UCgYEA7oTm8/k1ZH/3aDONmBu3gSDcYFDfyHiMCZjjF+wcAl/1RfRniHN9
14+
ZwpHXpqewkojKGaIxkDFYMYcRPVCvxZpKTTzIbHmdWL2zMa4tOd8/vtAgwiaRTxz
15+
CWpkuitTqhMKRZMuk6GLN2aJGhW4xoztoxBwvwl4hIcOF/nDUx5wGg8CgYEAw4ju
16+
TywfXLVukeGzF2NR/RHafv5lIGFVAXz6S828gVdAd723Ar5KfUcuHSWou6jeSLxC
17+
X6moxNXg9pzemj5oOxixm5gNmMhhGc5saofVF+Yw7OXiO5HS48N5/y8UhLzG9Woe
18+
woIzLOedI7LL4c5k63Dv1PBRfF7SgotUD9o+cjsCgYBTlIStl3UrO2BBxlbmp0e9
19+
Wvt+ULiqMfu4XTeuqVEDUAbNtaAWA4o88jAiKYJaSMpbHNruxcGkcKd1TpBOD5dY
20+
FjLgast79OrQxQ3ybm+xle98OyZ7/nu9H+geCJKvtS5TRQj8OI5lImwGlPgDs1yO
21+
WHN5reLqZbz9j8rvNGpJ5wKBgHNduBU4WCyv01P5EVxsOjSRMxQYbnwDUU82eMr+
22+
0QiS/TNo2LCiq3ps49VIodxOzw+lG5grlqrD8h4vAduJlJ/BexDtTqBlIX1DYI4e
23+
JGkcZKNxIuC4se+2JqWcFoQWiWDEwu9KOH13DizA0jAhQUP0d1xwifNSUuu43tkX
24+
kB5/AoGBAMbisz4YfKjxSieEPHz7F4GiIhCe+Xbvj2AwkantBcKNdTmt7uHoE1BM
25+
lGqKz1uRhliBPrHcbXQnLr9pKs5XvbgD67CM8461ZTRSA8hjV+4yBl3oLtoP5kc4
26+
MUSyPfkASNQzd7VSH36rdLFu1cM1OKe+L1Cr7s81S5LaWplsJoXP
27+
-----END RSA PRIVATE KEY-----

‎resources/dvc_playground/user/resources/inputs/parameters.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"row_shift": 47,
3+
"col_even_shift": 17,
4+
"col_odd_shift": 1065
5+
}

‎resources/dvc_playground/user/resources/inputs/part1.input

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
%&.¨§D£}X{!?$€X{?°r}?{ddD¨"°X}£{[/&@%/.r{{€§![£$[d°rr!d!rd?[}€¨?£$?./%&@@%#/,.r{,%@%,°!"[£D£€$Xr}$°{?[€[§?"§€}"£¨[{{[{{°D}d€.&@@@&/(,}(@(,@,*/§¨[r§§$[?r¨£?}r!°*
2+
r¨§§€?Xdr§.,/#&@@@%/,€r{¨}[DD?!°¨£r,&@%,r°§"¨X"?D¨€€"!?[{£€£D¨D[r[$!€!X°D£€"/@@@#{r!}}&#.*@.*/£$X€€}°![rdX€$[°!°}/@/[X°£}"°°X[}§X£}€r§€{""!$.(@@%*[$D¨{"D}X!D}X§
3+
[}/@/£$§€[r§¨¨}r°$€}€$¨X"}*&@#.X£°rd!rX"X{d?¨$°$§§"[D./%&@@%(*.D{"X¨XDDr{"X{"DX$¨X€$.#@@@(.$£§€X$£¨¨£d£$°[}¨€$d°r§"rD€"[§!XX*@@*!¨r[d.@/€*@.*(¨?°!${d?°?£!¨§!r$°
4+
#&@&%/.!r$X"r"£¨$[€°!€D°[$°X}¨X§"Dd}d}¨*#@@(.?¨"°[£r!D![X°$d¨[{X€"}X}XX?§?r/@(?!€!$$X,@/[*@..(¨X°${°}€!$£§§§dr!X°D$.&&.rr°r[?r["rDX!X£,%&&/r€"X°£§¨€$€[}$"dd}.,(
5+
°?"¨°/@(.[§X[{¨{rD$*&@#.$°§?"r§??[${dD"*(%@@&#*.[§[!$r?§$r"[?{d"°r"[!rrdr"?°¨[?D¨°{§¨§r§§£(@@/¨r}¨D{°?drr!"€d!€°dD°€d?!}d}"..}[?$?d?!.&(.*@.?($!X§X[XXd"[r{°$[§$
6+
}X€?r°£€"£§¨€D}[°r?"D?°D£!$X?r[¨D{€£Dd£€r![%&.r¨}!¨{}r{$r££"?$}}}d§d$?dr"?D{r}{Xr"€X€"*@%/@.{/?€[dD{X[D!{{}€€°°d[r!D}£.#@%.{[§d,/@&/X}$¨{rDDX§{§D§,#&@%*,{$r£[!d
7+
$XdD§D[€.%@*€/&@%,?$€D€°€¨?$[X*#&@&(.${$rdD{!X}DX¨d$$"¨r€§Dd§$"{°°rrr$}d¨£$DX€d{$§??D?$"X°,@($!$!}?¨£}X$§"X{d}{{r!}}°D["r£?"§?X§"}[d§¨€(@@@.€*?r"¨}£°¨[[d°X$¨°$d
8+
¨°{¨{X{Dd°?d§¨§X!r¨"£D€r§¨"D!rdd?D¨Xr§[¨§,@&."{°?$!°X}D€dd$§!X}[!{Dd"€${§£dX[X["!§"$§{?€.&@.D.}§d§°}{XX€Xr$€°[}°"Xr¨}D¨r¨!.%@@.€}}d€$"d.,(&@&#*.?§["°DDd$X!!dD°¨
9+
°§£${XD!°§£€.%#@@@@@@@%(*.§""[Dr£¨!¨D§[¨D}£D}}r¨"$€D$["X£?Dr{°£?£?$D¨?$£{€X[$$Dr§rX¨£"dD.&%°$¨¨d[?£€£r¨°X??!§¨r!r{{"X"}§"$!§["[£[!X[€X€°°,@&.{°}°r£¨[d$XX}X[}$£}
10+
}}€€{dd€?!£¨£XXrrd§}$X¨}€°§€!![?Dd°{"¨[*%@,X!§€{{$°!€d€{!XX§{!$£r}D$"D}D[€§€"[}d!Dd?$X€°{£&@@,£§d$$€¨r£!}?X€}{}X}"D£D!§X°¨$"r£?¨{[$€}r[?{drd¨?r"dX?€€§${£[§"{d§X
11+
![}drdX""£!!!{d"§r{¨$?§{{[XX£°{¨d!X{°"$¨§¨[?!{"[}£§Dd[X$$X}$°¨!X£¨"¨¨dD!§€}D°}$r°D["d€[%@(£!!$}!{"{d$§$[?"?€"$?[$"X[°{$D!DX"°{d£?§?¨?§[?"$(&&&}!£€$§£{?$d°XD$r$£
12+
DrDrd£€¨!!$[$"[d!D¨!"[}*(@@@@@@@@#(,.°"?rd°X[!{X""d£?....%&.?Dr!X€¨!DX$£°[D$[$£{[?dDD!{"°d?d£"$°!£X$°?[?$[°}€£§[£"}£¨D?r§°§€{{X{€?Dr§°$D}!}°£¨"X}}¨D$°£d§¨§}?"?$
13+
€}£°D"r€[¨°!!X$°¨!d?"d}d[°D$§$r"¨§{£"!¨§}€?XX°"§?§D?"¨d¨{{r£§}r§°££.*(&@@@@@@@@@@@@@@%*!§X"}$[£[$r[,%@@@@%@#°€£"$[?££!¨d{X!°r°°°?}r[°°X€¨£d"€{$!"§[{"€§?¨d?€rr°!
14+
D€€?£§"[d?§{r§r".#@@&%(*,/(#####(##%&@@@@#.Xr?¨§r,&&,{£[X(&@*€¨}{r"Xr€}§€§$D"r{D"£¨£dXr€?{"D$$€¨?$d?°°d}€°!€§}!{€r}[¨€[§£X{£}§€"€?"°€§{"¨£D§?.*#####(,$}XD!dD§€D
15+
°§d?€$rr"¨""Xd[°"§"$£$""r(&%/#########(....£?!?"?¨r°$€d!$°!r€°(@@###//#######*,....,#(#@@%@&,$}./@/£$£XD°¨[#&.[£X£[X[}Xd?§XX[r}§}X"?"dr§"DDX£}r§€"§!d{[§?[!!€}}r
16+
"§[€?r§D€?§"*@&###%##(((((/***/((((((((((&&(@%*@&,X§[$?rD§§.&(dr¨!¨!{X[£§"€$$X?!$¨£!!°d§£X€{°[¨r[r"!!§?"°X"$r°rr¨°$?!Xd§$}!{$¨¨{§D¨{€d!D"*@%&@@&%####((((((((,$?
17+
r°![§"{}°"[[${dD?[?{£D[§?,&&##%&@@&#((((((((((,}r{{{?{rXd€.#@&####((//(##((((((((/**,/(/(#@&*&@&/}dr?d{}d}r"(&.?d{d£D?}¨r{£${d"$§X$rrd£°[¨d$r{?[d°D¨¨€£?XD}€!dD"
18+
,.€€§?"°.(@@###(((((((((((/,.r€[{..,,**////#@%@@#"°dX§£"€$D{.&(€D€r§[}D€$£¨§"$Xrd¨!dd£¨¨!D$D°$§°}°§$$$¨£¨}¨!°r}rDr!?{D€[r$£¨Xr$§{§?£?d?§§¨*@%#((((#&@@%#((((///*
19+
$d!rD}${dr$X}!}$¨[}}d¨}d?{.#@%((((((((#&@@&(/******,./&@@@%#(((#((((((*..[{X.,*****/#(***,.,,*/.£°£D¨{€{¨¨!}"(&.?°[D€D"€}{!¨Dr$§}d€D{!{$"D¨}§§¨X°r£!d"?d?°d"£$€D
20+
/****/,¨X,(((((((((/.D"?.,********/##(###(,&%&%.,d£D¨?{X§§"X[.@/X€}}$?$DD[!{£¨r?DX[D§}X$DD°$$¨{€[d"°{£[¨rr{§§°}XX¨£"{§X£§$¨?}°{§£r£{$££X°r$.#@&(((((((//**/#&@&(
21+
D[¨Xd!dd[£€§{"D"$§°X¨{D£{£[}.#@%#((((/*******/#&@@&%%#(*/((//(////,¨?€,**********,(######/.&%&@@&,€{£€§r£§?$€£%%,"!¨$XD{£¨D£[D"¨?§¨D!?[}{[[£D{€"°r$[dXd[§"r£X§"[
22+
%%%@@@@@%(.//*(*,}§r,********,,**(#####(((.&%&@#,!$D},*¨°¨°}!€*@@@@@@@@@@@@%#(.°[¨}¨?d!X{°!XX}$}["§["{!¨[¨€!¨D!"}$§D"Xd[}[}??{??£!€¨[?€{!X?r§,#@@#(/**********/%
23+
["DX§}§X¨{X$°dX{€r£{D[!€"€D£}*((#@%*******/(#%%%%%%%%%&@@@@%(/.¨€£,****,,**.,**/(#(#######(@/%@#.£DdD/@.D¨¨£D$"&@(%**,,**,..*#@@@&/.[§DD?£$Dr"D"£°r?"rXr[€€¨€[?!
24+
%%%%%%#((#&@@@%/****,.**,.**../#(/###,§*(#%&.%@%,¨°}r.&/.$¨{$!€/&#?,*,,****.D,*,*@&(#,¨D"DrX€X{"¨}¨{$¨§€°[[¨d¨?¨}${[£X{D¨¨°¨DX££r€§d"$°D[X$"r./(#@@@@(**/#%%%%%%
25+
d£d}[[{}?€d£{r!!"DXrdd!§°d!$X!.*/&@@@@(#%%%%%%%%%%%%#((((((#&@@@@##(/*****,£X*#((###,",##*@%°#@@(rX!¨d%%,£!£}€?"/@,,*****,*,,**,..#@@@/Drd"€r}}°€Xr"§d?X"$?"??€r
26+
#(((((((((((((#%&@@@&#(((,[£.(####(d§/##&@@@,.&@@.?rd{/@#!£§"DDX[%%.***,*,***.*,.D,/&@@&(§D£}€dXd}¨°£[dD!Dd€£€Xd""r§[€§$°$r€??§X¨??€rr"{?d{€D°d"¨(@*&@&%%%%%%%%%
27+
X!°XD$¨?!}d}!$?¨€$€°°"X}°{$X€rD€!*@%%@@&%%%%#(((((((((((((((((((((#%@@@@@#€,######&%###@@@@@%X,&@(r}D{"%@,£°$[!£€.@(,**.,,,**.**,€,*.*@@@(}D¨{?£§}D{}°d£{}"?}dX{
28+
((((((((((((((((((((((#%&@@@###(,*#*/#@@@.?.&({.(&.d{?",%%["}$"Xd{*@***,.*,**.**,!,*.[.&@(,?}[§?[££§€{£rD{{£°[!$£?{}£{}"X?[[}§"{{°°DX€§X[X§££!D"!,@&%@@@@%#(((((
29+
£[d!D°[§°{€¨°€{}[$£°£r§D°"$X€$"![,@&@@%&&@#(((((((((((((((((((((((((((((/£(%##/,/(.,##@%*D¨D/@.[.@(€Dr""*@,§X§X!X{X%%,*,.,***.,**.,*.}r,#@#.°}[X?}¨d$¨§r[D{£d°}D
30+
((((((((((((((((((((/,X?¨¨[(#/(/,€(@@@%d!dD}{%%"€*@,"{}r¨#&[{"£{"[?.@#**,.*,*,,**,.*."r€.%@*d?°d"!X¨d€D§{"{r°X?§€§£Xr}{ddX{£$r}¨d°{X$°€§€}"}[}rD.(@&##%#(%@#((((

‎resources/dvc_playground/user/resources/inputs/part2.input

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
§¨[[D¨rr![€![§$!dr£"?§!°XX!D./&@@%/,###(((%@%((((((((((((((((((((,[¨DD??.#@%%&@@@@#{*@,[£€$dX.@#,£(#?€"?r*@/£[{}Xd[D,%***.,*.*,***,*.{€.,*&&.!°¨£}XX€r"$"{"r}[¨[
2+
(((((((((((#%@&#(&@@@@@@@@@@@@/(#,°,@(X?"[}£"!€%@("[§€§£$¨,#.¨[{¨$r§",(**,.,,,******.¨[.,*&@#!?r}dr°$dD?¨XDX$r€§?D}€!£r$D€?°X![XD}§!d.,/(%@@%/#&&#####,/(((#@&#(
3+
X£[$€€![}?"r.,,/(%&@@@&%(*,,(########,/((/(#&@@%#%%##%%%%&&@@@@@@&@@@@@&&&#%%###/*(@%r§X€€[€§X{D*@(X$d"€r$}?¨"°£{Dr!?°****.,*.***,**,[{,**&@@&}r§°X$rd[!Dd£r§{[?
4+
#%%##(,..,&%/@,.€rD§!D,..,,,,,,,,,.r"d?r}{{DX}}dX"%/£d{!D§£r{§£d¨¨?[X{$//**.,,,**,,**.?,*,&@@@dD}°£"X""£{d¨[€€¨$£§¨[?€?,#@@@&%#//***/((############(,*(((,.%@&%#
5+
!D?$$rrr£$.(@@@&%%##########(,*###,*((*/(,*@/.***,.?(&.rD€.&@@@%*}¨D€§D°r{"D{{X¨D€?D"?X§°°€°[?¨€¨°"XDd"¨"XX?§XX{°{r{¨d[{*&(*,*,,**.**.°,*,&@@@{}!€D$"X¨$![°!dD}r
6+
$/@&*§[°£{*@@@/€?{§¨¨}X$${"{¨°XXD§"?€!{!?{€$£![rD{¨€€X$$d?D°"d??$?!r?}"?X&@@&**,**,,*,},*,&%@@[£{§[°§!{d§"D{[[£d[[DDr$D€[!""{,%@@@&(,*///*,.*#((((((*.,(/*@/}{}d
7+
€XrXddXdd}£°°"?,/&@@@@&##((((//(#%%,§.(#/&%{€${#&&,{X$}¨[#@@&,d{¨dD€!"X$D!}"[d§"$rDd£?§€€?d¨?!d[§£{°["d€€£D°€{{rXX§XX§€.*@@@%*****,.**.,**&%@@d€X£D{"Dr}![£§r{?"
8+
.££"}/@@@@@*X¨°°"€?$§r[$!§°¨ddX!$§£r°£°}${r£[€€[£}$X?D{°°$[£d{{[dDd¨#&(@&#@&,%#**/*,,*****#/&@[§!{"!€!°€¨{!r?D$ddX"¨"!DX!{§X§°$}Xr}.*(%&@@@&%##@&.[?,%@##@.d,&@*
9+
?!}££dd?d?X"§{"£[€?€"drd{?!.*(@@@((#@&.*#%%@#.€X¨$D°*@@@@/?¨"[X°??r€DD"°¨D£€°r°[¨}"}?r¨{!r£€£dD!}{X$°€?€!°r°r}[d€§"r&@%/%@@**./&#/(*,***,(#/&@€d}£X![[}}d?X$D¨"D
10+
¨€,@@@@@@/X¨[€rr}X{°?{"XX¨D°£€}X}X?"X{D[§¨D!€??{¨"dX"[[}d€{°"dD$}!?,@%/@&,./(/..*#(*%/**,%#/&@{{{€€€d£X?}{€°"€°r"°°!¨°D{X€?["$r€€D"¨"€§$d"£d.#@&/,%@@@@@&/"£XX$$
11+
D"€XXdXr°?§[}D!?[!D§[?§${!§,&&@@@@@#/,}$Dr?Dd¨X°r#@@#@@@@%[£}£d""}°![?!$?X?°¨£""°€$?¨§?[§€{X}?d!££§°€}?§[}!?[d!{§"D/@/@#,}¨((/.".((*#&/*/&#/&@Dr¨d¨$°£?{D$?§dX![
12+
@@/$.@@@@@*[""§€¨X"£d[{["?$d€$!¨¨DD"¨€!r§r°!}!D?£$?!$d£§€[!"£€¨°X?€%@@#*((##(#(..(*(&%@//@#(&&?r°£{}"d}¨£¨£$"{$°r"§¨{}}€}r°D"!D£r{{{£"D?¨.(@&&@@@*}?!"§°[D$}r".&
13+
r£d[§?{}XrDD}¨r£rX{X?°X.,#@@(*.[£¨}°[{€°!°{rd#@%/!D€.@@@@@@.{rrr$¨dD!°[°D"€r}?X}[°£"D!"€X[¨£"!?XDD}?¨{d![r§X€{{}D§.@@&£d,(((((/@&#*[/@&@@&(#@#D?¨¨€$X{{["§£}§}{€
14+
$!dd.&@@@@&dD§}r€°§¨}X"D}§""[§?D?°d{r§d{r°§[£$rdrrD°€D[£rD£!{{"¨!d*@@/"?X/((((.X,(%*",@@@&/#@/D?}!rdr¨?{£"X!¨§d}{d{XdX€!D€{[DdDX$D{§*%@&#,"dD§}D§d}}!°[§€?}(@#}D
15+
D!}?!£{dX[}€¨X¨¨r.(@@(.£}§£§d°€¨£"!¨!"§¨§¨(%*Dd$D£[}[&@@@@&.°£!?€!¨$$££¨r}?¨Xd?§d€X"£$"°{r$![?Dd£[d!d[$X$€r°rr$dDX.@@,£!§,(((/.£.(%@/,#@@@%%@(!¨§€rD{}[¨¨§£"$$$!
16+
r£§[?(@@&&%!D{D€¨[!Dd{{?r{r{¨["d°$Xd£d}¨¨X"!X°DD[°"?${€[d!}!€D§rd€$&@(D!§!/(((.$.(*.&%//&@@#((}D§?°![£{{dD"¨§€X[¨°!{°[}r!¨[}°./&@%*§D[€}}[$r?r°rD°"}?XD{$Dd€€{D}
17+
£¨£D€??!}{°*&@/€§}€°}°DX{$}°$°dr€?£Xrrrd"d°§€}?[$$$€?,@@&&&}}€}dDX§r£"D!{}§drrdD§°£}$°[$?"{$!DDr¨[$°d?°°r?°£[XdX}$.&@@@*.X,((/.¨.(*?.(%#&@@#/(X""$}}{¨X?€¨{?"![d
18+
}dX¨"°#@@@&.$°DdD?°"£°r!d{{["XX}X¨?$}[{§}X€}"d[£XD!d£!dd¨°!D"[$[$X.&((@@@&@#((.d.(*§.(#@@&%%##£!€$[r{${d¨¨D€€X§?X§°{}€"X.#@#.}°!X"¨{??{d¨Dr§¨[}$}$!$"£D€!°DD[§{"
19+
[§°d}£*&@%.°D?[{°X§°dX$[X€{°§°$£°£[{?D§!°£?rD"dr£?"X°{¨%@@@.Xd£°D}"?[!{£X§°![X£$[°}[°"r¨D!d}$°D§€§£r}£°°!{{XDX¨}$r,@/X,%@@@%(/.¨.(/!,(,#@@/#((§$¨d°"!dDr{}}dD{£€
20+
}{D°¨D€D.##.{§r[€"§€"d€§?!!d§}¨"D?Xdd§}XX{X!$°?§"€€$§{"£r€°€€£!DD{,@/"°![.#@#(.!.((.,(.!.&%,((X?!£$§dd${}€!¨$!§[¨°d£(@#.!€¨§?D{?§$dd£?!£}¨?¨§d£°$X?"X?¨°?°£¨{£$¨
21+
X,#@#{!€![.#&*D°€"€¨rd§°§€X""°D}€Dr$§{X¨?"$¨"!}r£€"D{¨"[r°}€[D°°¨${!?$°dr!"€[£D¨$r[?$"°}["§X§!![§?XX}D°"¨{"r£"£{Xr,@/§[X€{?/@&%€}/(.,(."d.&&#({§€£d{[}"["d¨}}"!{
22+
$d}§"!§€r!¨[£}"¨![§DD$€}€${°§d$}!"}d[r€!£?D€¨$£X{{§dd$¨{!d}d$"X§}X/@,?£?.$d"(@&*[*(*,(.?£r,&#(r[}{£€{XD$£${${!?*#@*£¨}}d[,@@@,°??"[!d§$°}[$[D?r{§XDd£¨€¨[$€$Xr$°
23+
&.Dr€£!}¨,@@&?d$?$§[}{"€X££{d{°!X"X}§§¨¨DDrX¨{}€¨{£r§D!X¨£§°!$[X¨£?D§[€¨X?€[€$!£"§?€°£?r§Dr§§}D£[{?¨r$$"!d£}§£!"XD.,/(//*¨"[.&@%*,(/,(../@@@#([€!D}!??"[€¨$¨X€/@
24+
}d£€D{"{¨¨$°Dd£D?r"{d{°$d}$"!X¨£X°[dXX?!d°$d{}${$![!$?}¨XDd.*(%@@%*€XXd$?¨}D°*@@%.((*#&@@*(@%(d££"§$?}dDX£!}?,@/.!¨${$rdr(@@,r§}}¨°!°°{[X[r[$§!¨¨?D{rd§"§}§€X"£"
25+
XXD{§d}£°&@%°§$€$D$r£r{€X$€$}D[r}$§§°dr!["$¨r§[""§d,(&@&&%,X!d{"€¨"r$}!°D?"{r!£rr$"£r€}[$?£{"¨["Dr}£,(%@@&#*,.°£¨"}[§§"!§°}"D¨&@#!/(#&@,¨.@&((€"[X!{dD[d§!}r{/@,
26+
&@%*.}}!{!¨r}r€X¨[$€$DD!X[Xd£€X$§!r"°$[?€¨$",/#&&&%(*.rX£d¨€{$€X¨$}{rd"£¨¨!€D?%@%*(#@&.}.@@%*(r"!¨"°r?X!"§}{[/@,X{?d"¨r.(@@(¨$"!°£XD[}¨°!€r!§"D°X}§°¨¨r[{?{}"./%
27+
.¨§X"Xd/&@@*drX[{?dX[[{?r?"!£€?°D}°€€?[?}€,(&@%*.D[€$"d$§"rd{[d$$¨¨!?{£d[§£"£¨rX}X§€{.*(##%(*,.D"£§[{D!£!€d°£X£"X£°!}§!!{r$D$D%@%#&@&(.,#@@#*(¨¨d?D$}§°¨!?[{D,@(
28+
?}°rD}§rr}§£§"?£$DX"{!DXX?d.*(%&@@@&#/,..£!{}¨°!rr"D$?[?D}£r¨Dr"dd}{?¨[XD?D$d,@@@&@%,[¨&@*@/*(?$DDD!£"{€$£"X!£*@&[[}€{D!..*.}[r$§DdD"¨££r$!°£$!!"r}$?,#@@&/.!{}d

‎resources/dvc_playground/user/resources/steps/concat_files.py

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env python3
2+
import glob
3+
import logging
4+
from argparse import ArgumentParser
5+
from os.path import isdir
6+
7+
if __name__ == '__main__':
8+
parser = ArgumentParser(description='Concat files from a directory')
9+
parser.add_argument('-i', '--input-dir', required=True, help='Contains files to concat')
10+
parser.add_argument('-o', '--output-file', required=True, help='Result file')
11+
12+
args = parser.parse_args()
13+
14+
if not isdir(args.input_dir):
15+
logging.error(f'Not a directory: {args.input_dir}')
16+
else:
17+
with open(args.output_file, 'w') as fd_write:
18+
for file in sorted(glob.glob(f'{args.input_dir}/*.input')):
19+
with open(file, 'r') as fd_read:
20+
fd_write.write(fd_read.read())

0 commit comments

Comments
 (0)