Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dvc playground tutoial [WIP] #4

Merged
merged 1 commit into from
May 6, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions resources/dvc_playground/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
version: '3'
services:
remote_dvc_repo:
build:
context: ./remote_dvc
ports:
- "22:22"

remote_git_repo:
build:
context: ./remote_git

user1:
build:
context: ./user
args:
USER_NAME: "songoku"
command: tail -f /dev/null
links:
- remote_dvc_repo:dvc_srv
- remote_git_repo:git_srv
user2:
build:
context: ./user
args:
USER_NAME: "bulma"
command: tail -f /dev/null
links:
- remote_dvc_repo:dvc_srv
- remote_git_repo:git_srv
21 changes: 21 additions & 0 deletions resources/dvc_playground/remote_dvc/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
FROM debian:stretch
RUN apt-get update && \
apt-get -y install openssh-server vim && \
mkdir -p /var/run/sshd

RUN groupadd ssh_user
RUN useradd -g ssh_user -d /upload -s /bin/bash poney -p azerty
RUN mkdir -p /data/dvc/remote
RUN chown -R root:ssh_user /data/dvc
RUN chown -R poney:ssh_user /data/dvc/remote
RUN chmod ug+w -R /data/dvc/remote

COPY pub_key /tmp
RUN useradd -g ssh_user -m -d /home/dvc_user -s /bin/bash dvc_user && \
mkdir -p /home/dvc_user/.ssh/ && \
cat /tmp/pub_key > /home/dvc_user/.ssh/authorized_keys && \
chown dvc_user:ssh_user -R /home/dvc_user && \
chmod 644 /home/dvc_user/.ssh/authorized_keys


CMD ["/usr/sbin/sshd", "-D"]
1 change: 1 addition & 0 deletions resources/dvc_playground/remote_dvc/pub_key
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LtCP20Y3Dxy1I7lVB98PqoUuFS7ggeC3pHC32r8e8Vfwhj73TVSHPvdUQDNLWXt+NuZpdSFrpWoT6l1YMHyRZArC1QiM0t+4ptt8Fr1baupnXHO3I74gp89+XEMdDjqSR9WZ0MIX1KRV956samUuEdHmEMTw22HaUQBCrk1b2P9J7e5AchNXJWMWITq9Rorzg58Pquj3ejENKlotAAyVzTAwrnfUuTlmTC96GXarJp4Pkx4LlWJv4J18XfrBRfGUD5F23IBMiII9fQtxrQZ3ntb3TALGCFmxs8udT0eXefsej10iAvrllP5Qg70fNKHpD31qn570AirbT+4FEq91 sbracaloni@poney
19 changes: 19 additions & 0 deletions resources/dvc_playground/remote_git/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
FROM debian:stretch
RUN apt-get update && \
apt-get -y install openssh-server git vim && \
mkdir -p /var/run/sshd


RUN useradd -d /home/git -m -s /bin/bash git


RUN mkdir /home/git/.ssh && chmod 700 /home/git/.ssh
COPY pub_key /home/git/.ssh/authorized_keys
RUN chmod 600 /home/git/.ssh/authorized_keys
RUN chown git: -R /home/git/.ssh
RUN mkdir -p /srv/git/test_dvc_remote.git
RUN cd /srv/git/test_dvc_remote.git/ && git init --bare
RUN chown git: /srv/git/ -R


CMD ["/usr/sbin/sshd", "-D"]
1 change: 1 addition & 0 deletions resources/dvc_playground/remote_git/pub_key
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LtCP20Y3Dxy1I7lVB98PqoUuFS7ggeC3pHC32r8e8Vfwhj73TVSHPvdUQDNLWXt+NuZpdSFrpWoT6l1YMHyRZArC1QiM0t+4ptt8Fr1baupnXHO3I74gp89+XEMdDjqSR9WZ0MIX1KRV956samUuEdHmEMTw22HaUQBCrk1b2P9J7e5AchNXJWMWITq9Rorzg58Pquj3ejENKlotAAyVzTAwrnfUuTlmTC96GXarJp4Pkx4LlWJv4J18XfrBRfGUD5F23IBMiII9fQtxrQZ3ntb3TALGCFmxs8udT0eXefsej10iAvrllP5Qg70fNKHpD31qn570AirbT+4FEq91
24 changes: 24 additions & 0 deletions resources/dvc_playground/user/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
FROM python:3.6
ARG USER_NAME
RUN apt-get update && \
apt-get -y install openssh-client vim git tree && \
mkdir -p /var/run/sshd

COPY private_key /tmp/

RUN useradd -d /home/$USER_NAME -m -s /bin/bash $USER_NAME

RUN mkdir /home/$USER_NAME/.ssh
COPY private_key /home/$USER_NAME/.ssh/id_rsa
RUN chown $USER_NAME:$USER_NAME -R /home/$USER_NAME/
RUN chmod 600 /home/$USER_NAME/.ssh/id_rsa

USER $USER_NAME
ENV PATH=$PATH:/home/$USER_NAME/.local/bin/
RUN pip install --user dvc paramiko

RUN git config --global user.name $(whoami)
RUN git config --global user.email $(whoami)@example.com
COPY resources /resources

EXPOSE 22
13 changes: 13 additions & 0 deletions resources/dvc_playground/user/dvc_init_repo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/usr/bin/env bash

pushd ~
git config --global user.name $(whoami)
git config --global user.email $(whoami)@example.com

git clone git@git_srv:/srv/git/test_dvc_remote.git
popd


dvc remote add dvc_remote ssh://dvc_user@dvc_srv:/data/dvc/remote
dvc config core.remote dvc_remote
tail -f /dev/null
27 changes: 27 additions & 0 deletions resources/dvc_playground/user/private_key
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAti7Qj9tGNw8ctSO5VQffD6qFLhUu4IHgt6Rwt9q/HvFX8IY+
901Uhz73VEAzS1l7fjbmaXUha6VqE+pdWDB8kWQKwtUIjNLfuKbbfBa9W2rqZ1xz
tyO+IKfPflxDHQ46kkfVmdDCF9SkVfeerGplLhHR5hDE8Nth2lEAQq5NW9j/Se3u
QHITVyVjFiE6vUaK84OfD6ro93oxDSpaLQAMlc0wMK531Lk5Zkwvehl2qyaeD5Me
C5Vib+CdfF36wUXxlA+RdtyATIiCPX0Lca0Gd57W90wCxghZsbPLnU9Hl3n7Ho9d
IgL65ZT+UIO9HzSh6Q99ap+e9AIq20/uBRKvdQIDAQABAoIBAGuQ8pWtqySBKLsO
77Nz5VFTpjTSTw5BwRPM5Ly5TFURvJGIQg80Om9sTw4QpficSdz4v0yO2v+6+Anw
O9MBNduGTRBCRj0XmP5mjnsS1hTu6J3EqZH9aZnQWyPJh9rU9uI9xbF+38rAQ5N6
k9ZLL5hm7x6+0flAHuOPJe85CPXsBm8wRX7J7h9865uX+L7zvt7rRWBtGS5gqVj9
uRm+wwJiOzjc86Q1h725Ev7pKchYMtFPNIMmrrSQsdRinhDyNCPamZI2YISWNaHM
3814MbPi9M3v71YBFp80JAcaebgRO/enC4f/qUaaHtVtV65cgoaacLytdBf120n9
MqRLT7UCgYEA7oTm8/k1ZH/3aDONmBu3gSDcYFDfyHiMCZjjF+wcAl/1RfRniHN9
ZwpHXpqewkojKGaIxkDFYMYcRPVCvxZpKTTzIbHmdWL2zMa4tOd8/vtAgwiaRTxz
CWpkuitTqhMKRZMuk6GLN2aJGhW4xoztoxBwvwl4hIcOF/nDUx5wGg8CgYEAw4ju
TywfXLVukeGzF2NR/RHafv5lIGFVAXz6S828gVdAd723Ar5KfUcuHSWou6jeSLxC
X6moxNXg9pzemj5oOxixm5gNmMhhGc5saofVF+Yw7OXiO5HS48N5/y8UhLzG9Woe
woIzLOedI7LL4c5k63Dv1PBRfF7SgotUD9o+cjsCgYBTlIStl3UrO2BBxlbmp0e9
Wvt+ULiqMfu4XTeuqVEDUAbNtaAWA4o88jAiKYJaSMpbHNruxcGkcKd1TpBOD5dY
FjLgast79OrQxQ3ybm+xle98OyZ7/nu9H+geCJKvtS5TRQj8OI5lImwGlPgDs1yO
WHN5reLqZbz9j8rvNGpJ5wKBgHNduBU4WCyv01P5EVxsOjSRMxQYbnwDUU82eMr+
0QiS/TNo2LCiq3ps49VIodxOzw+lG5grlqrD8h4vAduJlJ/BexDtTqBlIX1DYI4e
JGkcZKNxIuC4se+2JqWcFoQWiWDEwu9KOH13DizA0jAhQUP0d1xwifNSUuu43tkX
kB5/AoGBAMbisz4YfKjxSieEPHz7F4GiIhCe+Xbvj2AwkantBcKNdTmt7uHoE1BM
lGqKz1uRhliBPrHcbXQnLr9pKs5XvbgD67CM8461ZTRSA8hjV+4yBl3oLtoP5kc4
MUSyPfkASNQzd7VSH36rdLFu1cM1OKe+L1Cr7s81S5LaWplsJoXP
-----END RSA PRIVATE KEY-----
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"row_shift": 47,
"col_even_shift": 17,
"col_odd_shift": 1065
}
30 changes: 30 additions & 0 deletions resources/dvc_playground/user/resources/inputs/part1.input
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
%&.¨§D£}X{!?$€X{?°r}?{ddD¨"°X}£{[/&@%/.r{{€§![£$[d°rr!d!rd?[}€¨?£$?./%&@@%#/,.r{,%@%,°!"[£D£€$Xr}$°{?[€[§?"§€}"£¨[{{[{{°D}d€.&@@@&/(,}(@(,@,*/§¨[r§§$[?r¨£?}r!°*
r¨§§€?Xdr§.,/#&@@@%/,€r{¨}[DD?!°¨£r,&@%,r°§"¨X"?D¨€€"!?[{£€£D¨D[r[$!€!X°D£€"/@@@#{r!}}&#.*@.*/£$X€€}°![rdX€$[°!°}/@/[X°£}"°°X[}§X£}€r§€{""!$.(@@%*[$D¨{"D}X!D}X§
[}/@/£$§€[r§¨¨}r°$€}€$¨X"}*&@#.X£°rd!rX"X{d?¨$°$§§"[D./%&@@%(*.D{"X¨XDDr{"X{"DX$¨X€$.#@@@(.$£§€X$£¨¨£d£$°[}¨€$d°r§"rD€"[§!XX*@@*!¨r[d.@/€*@.*(¨?°!${d?°?£!¨§!r$°
#&@&%/.!r$X"r"£¨$[€°!€D°[$°X}¨X§"Dd}d}¨*#@@(.?¨"°[£r!D![X°$d¨[{X€"}X}XX?§?r/@(?!€!$$X,@/[*@..(¨X°${°}€!$£§§§dr!X°D$.&&.rr°r[?r["rDX!X£,%&&/r€"X°£§¨€$€[}$"dd}.,(
°?"¨°/@(.[§X[{¨{rD$*&@#.$°§?"r§??[${dD"*(%@@&#*.[§[!$r?§$r"[?{d"°r"[!rrdr"?°¨[?D¨°{§¨§r§§£(@@/¨r}¨D{°?drr!"€d!€°dD°€d?!}d}"..}[?$?d?!.&(.*@.?($!X§X[XXd"[r{°$[§$
}X€?r°£€"£§¨€D}[°r?"D?°D£!$X?r[¨D{€£Dd£€r![%&.r¨}!¨{}r{$r££"?$}}}d§d$?dr"?D{r}{Xr"€X€"*@%/@.{/?€[dD{X[D!{{}€€°°d[r!D}£.#@%.{[§d,/@&/X}$¨{rDDX§{§D§,#&@%*,{$r£[!d
$XdD§D[€.%@*€/&@%,?$€D€°€¨?$[X*#&@&(.${$rdD{!X}DX¨d$$"¨r€§Dd§$"{°°rrr$}d¨£$DX€d{$§??D?$"X°,@($!$!}?¨£}X$§"X{d}{{r!}}°D["r£?"§?X§"}[d§¨€(@@@.€*?r"¨}£°¨[[d°X$¨°$d
¨°{¨{X{Dd°?d§¨§X!r¨"£D€r§¨"D!rdd?D¨Xr§[¨§,@&."{°?$!°X}D€dd$§!X}[!{Dd"€${§£dX[X["!§"$§{?€.&@.D.}§d§°}{XX€Xr$€°[}°"Xr¨}D¨r¨!.%@@.€}}d€$"d.,(&@&#*.?§["°DDd$X!!dD°¨
°§£${XD!°§£€.%#@@@@@@@%(*.§""[Dr£¨!¨D§[¨D}£D}}r¨"$€D$["X£?Dr{°£?£?$D¨?$£{€X[$$Dr§rX¨£"dD.&%°$¨¨d[?£€£r¨°X??!§¨r!r{{"X"}§"$!§["[£[!X[€X€°°,@&.{°}°r£¨[d$XX}X[}$£}
}}€€{dd€?!£¨£XXrrd§}$X¨}€°§€!![?Dd°{"¨[*%@,X!§€{{$°!€d€{!XX§{!$£r}D$"D}D[€§€"[}d!Dd?$X€°{£&@@,£§d$$€¨r£!}?X€}{}X}"D£D!§X°¨$"r£?¨{[$€}r[?{drd¨?r"dX?€€§${£[§"{d§X
![}drdX""£!!!{d"§r{¨$?§{{[XX£°{¨d!X{°"$¨§¨[?!{"[}£§Dd[X$$X}$°¨!X£¨"¨¨dD!§€}D°}$r°D["d€[%@(£!!$}!{"{d$§$[?"?€"$?[$"X[°{$D!DX"°{d£?§?¨?§[?"$(&&&}!£€$§£{?$d°XD$r$£
DrDrd£€¨!!$[$"[d!D¨!"[}*(@@@@@@@@#(,.°"?rd°X[!{X""d£?....%&.?Dr!X€¨!DX$£°[D$[$£{[?dDD!{"°d?d£"$°!£X$°?[?$[°}€£§[£"}£¨D?r§°§€{{X{€?Dr§°$D}!}°£¨"X}}¨D$°£d§¨§}?"?$
€}£°D"r€[¨°!!X$°¨!d?"d}d[°D$§$r"¨§{£"!¨§}€?XX°"§?§D?"¨d¨{{r£§}r§°££.*(&@@@@@@@@@@@@@@%*!§X"}$[£[$r[,%@@@@%@#°€£"$[?££!¨d{X!°r°°°?}r[°°X€¨£d"€{$!"§[{"€§?¨d?€rr°!
D€€?£§"[d?§{r§r".#@@&%(*,/(#####(##%&@@@@#.Xr?¨§r,&&,{£[X(&@*€¨}{r"Xr€}§€§$D"r{D"£¨£dXr€?{"D$$€¨?$d?°°d}€°!€§}!{€r}[¨€[§£X{£}§€"€?"°€§{"¨£D§?.*#####(,$}XD!dD§€D
°§d?€$rr"¨""Xd[°"§"$£$""r(&%/#########(....£?!?"?¨r°$€d!$°!r€°(@@###//#######*,....,#(#@@%@&,$}./@/£$£XD°¨[#&.[£X£[X[}Xd?§XX[r}§}X"?"dr§"DDX£}r§€"§!d{[§?[!!€}}r
"§[€?r§D€?§"*@&###%##(((((/***/((((((((((&&(@%*@&,X§[$?rD§§.&(dr¨!¨!{X[£§"€$$X?!$¨£!!°d§£X€{°[¨r[r"!!§?"°X"$r°rr¨°$?!Xd§$}!{$¨¨{§D¨{€d!D"*@%&@@&%####((((((((,$?
r°![§"{}°"[[${dD?[?{£D[§?,&&##%&@@&#((((((((((,}r{{{?{rXd€.#@&####((//(##((((((((/**,/(/(#@&*&@&/}dr?d{}d}r"(&.?d{d£D?}¨r{£${d"$§X$rrd£°[¨d$r{?[d°D¨¨€£?XD}€!dD"
,.€€§?"°.(@@###(((((((((((/,.r€[{..,,**////#@%@@#"°dX§£"€$D{.&(€D€r§[}D€$£¨§"$Xrd¨!dd£¨¨!D$D°$§°}°§$$$¨£¨}¨!°r}rDr!?{D€[r$£¨Xr$§{§?£?d?§§¨*@%#((((#&@@%#((((///*
$d!rD}${dr$X}!}$¨[}}d¨}d?{.#@%((((((((#&@@&(/******,./&@@@%#(((#((((((*..[{X.,*****/#(***,.,,*/.£°£D¨{€{¨¨!}"(&.?°[D€D"€}{!¨Dr$§}d€D{!{$"D¨}§§¨X°r£!d"?d?°d"£$€D
/****/,¨X,(((((((((/.D"?.,********/##(###(,&%&%.,d£D¨?{X§§"X[.@/X€}}$?$DD[!{£¨r?DX[D§}X$DD°$$¨{€[d"°{£[¨rr{§§°}XX¨£"{§X£§$¨?}°{§£r£{$££X°r$.#@&(((((((//**/#&@&(
D[¨Xd!dd[£€§{"D"$§°X¨{D£{£[}.#@%#((((/*******/#&@@&%%#(*/((//(////,¨?€,**********,(######/.&%&@@&,€{£€§r£§?$€£%%,"!¨$XD{£¨D£[D"¨?§¨D!?[}{[[£D{€"°r$[dXd[§"r£X§"[
%%%@@@@@%(.//*(*,}§r,********,,**(#####(((.&%&@#,!$D},*¨°¨°}!€*@@@@@@@@@@@@%#(.°[¨}¨?d!X{°!XX}$}["§["{!¨[¨€!¨D!"}$§D"Xd[}[}??{??£!€¨[?€{!X?r§,#@@#(/**********/%
["DX§}§X¨{X$°dX{€r£{D[!€"€D£}*((#@%*******/(#%%%%%%%%%&@@@@%(/.¨€£,****,,**.,**/(#(#######(@/%@#.£DdD/@.D¨¨£D$"&@(%**,,**,..*#@@@&/.[§DD?£$Dr"D"£°r?"rXr[€€¨€[?!
%%%%%%#((#&@@@%/****,.**,.**../#(/###,§*(#%&.%@%,¨°}r.&/.$¨{$!€/&#?,*,,****.D,*,*@&(#,¨D"DrX€X{"¨}¨{$¨§€°[[¨d¨?¨}${[£X{D¨¨°¨DX££r€§d"$°D[X$"r./(#@@@@(**/#%%%%%%
d£d}[[{}?€d£{r!!"DXrdd!§°d!$X!.*/&@@@@(#%%%%%%%%%%%%#((((((#&@@@@##(/*****,£X*#((###,",##*@%°#@@(rX!¨d%%,£!£}€?"/@,,*****,*,,**,..#@@@/Drd"€r}}°€Xr"§d?X"$?"??€r
#(((((((((((((#%&@@@&#(((,[£.(####(d§/##&@@@,.&@@.?rd{/@#!£§"DDX[%%.***,*,***.*,.D,/&@@&(§D£}€dXd}¨°£[dD!Dd€£€Xd""r§[€§$°$r€??§X¨??€rr"{?d{€D°d"¨(@*&@&%%%%%%%%%
X!°XD$¨?!}d}!$?¨€$€°°"X}°{$X€rD€!*@%%@@&%%%%#(((((((((((((((((((((#%@@@@@#€,######&%###@@@@@%X,&@(r}D{"%@,£°$[!£€.@(,**.,,,**.**,€,*.*@@@(}D¨{?£§}D{}°d£{}"?}dX{
((((((((((((((((((((((#%&@@@###(,*#*/#@@@.?.&({.(&.d{?",%%["}$"Xd{*@***,.*,**.**,!,*.[.&@(,?}[§?[££§€{£rD{{£°[!$£?{}£{}"X?[[}§"{{°°DX€§X[X§££!D"!,@&%@@@@%#(((((
£[d!D°[§°{€¨°€{}[$£°£r§D°"$X€$"![,@&@@%&&@#(((((((((((((((((((((((((((((/£(%##/,/(.,##@%*D¨D/@.[.@(€Dr""*@,§X§X!X{X%%,*,.,***.,**.,*.}r,#@#.°}[X?}¨d$¨§r[D{£d°}D
((((((((((((((((((((/,X?¨¨[(#/(/,€(@@@%d!dD}{%%"€*@,"{}r¨#&[{"£{"[?.@#**,.*,*,,**,.*."r€.%@*d?°d"!X¨d€D§{"{r°X?§€§£Xr}{ddX{£$r}¨d°{X$°€§€}"}[}rD.(@&##%#(%@#((((
28 changes: 28 additions & 0 deletions resources/dvc_playground/user/resources/inputs/part2.input
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
§¨[[D¨rr![€![§$!dr£"?§!°XX!D./&@@%/,###(((%@%((((((((((((((((((((,[¨DD??.#@%%&@@@@#{*@,[£€$dX.@#,£(#?€"?r*@/£[{}Xd[D,%***.,*.*,***,*.{€.,*&&.!°¨£}XX€r"$"{"r}[¨[
(((((((((((#%@&#(&@@@@@@@@@@@@/(#,°,@(X?"[}£"!€%@("[§€§£$¨,#.¨[{¨$r§",(**,.,,,******.¨[.,*&@#!?r}dr°$dD?¨XDX$r€§?D}€!£r$D€?°X![XD}§!d.,/(%@@%/#&&#####,/(((#@&#(
X£[$€€![}?"r.,,/(%&@@@&%(*,,(########,/((/(#&@@%#%%##%%%%&&@@@@@@&@@@@@&&&#%%###/*(@%r§X€€[€§X{D*@(X$d"€r$}?¨"°£{Dr!?°****.,*.***,**,[{,**&@@&}r§°X$rd[!Dd£r§{[?
#%%##(,..,&%/@,.€rD§!D,..,,,,,,,,,.r"d?r}{{DX}}dX"%/£d{!D§£r{§£d¨¨?[X{$//**.,,,**,,**.?,*,&@@@dD}°£"X""£{d¨[€€¨$£§¨[?€?,#@@@&%#//***/((############(,*(((,.%@&%#
!D?$$rrr£$.(@@@&%%##########(,*###,*((*/(,*@/.***,.?(&.rD€.&@@@%*}¨D€§D°r{"D{{X¨D€?D"?X§°°€°[?¨€¨°"XDd"¨"XX?§XX{°{r{¨d[{*&(*,*,,**.**.°,*,&@@@{}!€D$"X¨$![°!dD}r
$/@&*§[°£{*@@@/€?{§¨¨}X$${"{¨°XXD§"?€!{!?{€$£![rD{¨€€X$$d?D°"d??$?!r?}"?X&@@&**,**,,*,},*,&%@@[£{§[°§!{d§"D{[[£d[[DDr$D€[!""{,%@@@&(,*///*,.*#((((((*.,(/*@/}{}d
€XrXddXdd}£°°"?,/&@@@@&##((((//(#%%,§.(#/&%{€${#&&,{X$}¨[#@@&,d{¨dD€!"X$D!}"[d§"$rDd£?§€€?d¨?!d[§£{°["d€€£D°€{{rXX§XX§€.*@@@%*****,.**.,**&%@@d€X£D{"Dr}![£§r{?"
.££"}/@@@@@*X¨°°"€?$§r[$!§°¨ddX!$§£r°£°}${r£[€€[£}$X?D{°°$[£d{{[dDd¨#&(@&#@&,%#**/*,,*****#/&@[§!{"!€!°€¨{!r?D$ddX"¨"!DX!{§X§°$}Xr}.*(%&@@@&%##@&.[?,%@##@.d,&@*
?!}££dd?d?X"§{"£[€?€"drd{?!.*(@@@((#@&.*#%%@#.€X¨$D°*@@@@/?¨"[X°??r€DD"°¨D£€°r°[¨}"}?r¨{!r£€£dD!}{X$°€?€!°r°r}[d€§"r&@%/%@@**./&#/(*,***,(#/&@€d}£X![[}}d?X$D¨"D
¨€,@@@@@@/X¨[€rr}X{°?{"XX¨D°£€}X}X?"X{D[§¨D!€??{¨"dX"[[}d€{°"dD$}!?,@%/@&,./(/..*#(*%/**,%#/&@{{{€€€d£X?}{€°"€°r"°°!¨°D{X€?["$r€€D"¨"€§$d"£d.#@&/,%@@@@@&/"£XX$$
D"€XXdXr°?§[}D!?[!D§[?§${!§,&&@@@@@#/,}$Dr?Dd¨X°r#@@#@@@@%[£}£d""}°![?!$?X?°¨£""°€$?¨§?[§€{X}?d!££§°€}?§[}!?[d!{§"D/@/@#,}¨((/.".((*#&/*/&#/&@Dr¨d¨$°£?{D$?§dX![
@@/$.@@@@@*[""§€¨X"£d[{["?$d€$!¨¨DD"¨€!r§r°!}!D?£$?!$d£§€[!"£€¨°X?€%@@#*((##(#(..(*(&%@//@#(&&?r°£{}"d}¨£¨£$"{$°r"§¨{}}€}r°D"!D£r{{{£"D?¨.(@&&@@@*}?!"§°[D$}r".&
r£d[§?{}XrDD}¨r£rX{X?°X.,#@@(*.[£¨}°[{€°!°{rd#@%/!D€.@@@@@@.{rrr$¨dD!°[°D"€r}?X}[°£"D!"€X[¨£"!?XDD}?¨{d![r§X€{{}D§.@@&£d,(((((/@&#*[/@&@@&(#@#D?¨¨€$X{{["§£}§}{€
$!dd.&@@@@&dD§}r€°§¨}X"D}§""[§?D?°d{r§d{r°§[£$rdrrD°€D[£rD£!{{"¨!d*@@/"?X/((((.X,(%*",@@@&/#@/D?}!rdr¨?{£"X!¨§d}{d{XdX€!D€{[DdDX$D{§*%@&#,"dD§}D§d}}!°[§€?}(@#}D
D!}?!£{dX[}€¨X¨¨r.(@@(.£}§£§d°€¨£"!¨!"§¨§¨(%*Dd$D£[}[&@@@@&.°£!?€!¨$$££¨r}?¨Xd?§d€X"£$"°{r$![?Dd£[d!d[$X$€r°rr$dDX.@@,£!§,(((/.£.(%@/,#@@@%%@(!¨§€rD{}[¨¨§£"$$$!
r£§[?(@@&&%!D{D€¨[!Dd{{?r{r{¨["d°$Xd£d}¨¨X"!X°DD[°"?${€[d!}!€D§rd€$&@(D!§!/(((.$.(*.&%//&@@#((}D§?°![£{{dD"¨§€X[¨°!{°[}r!¨[}°./&@%*§D[€}}[$r?r°rD°"}?XD{$Dd€€{D}
£¨£D€??!}{°*&@/€§}€°}°DX{$}°$°dr€?£Xrrrd"d°§€}?[$$$€?,@@&&&}}€}dDX§r£"D!{}§drrdD§°£}$°[$?"{$!DDr¨[$°d?°°r?°£[XdX}$.&@@@*.X,((/.¨.(*?.(%#&@@#/(X""$}}{¨X?€¨{?"![d
}dX¨"°#@@@&.$°DdD?°"£°r!d{{["XX}X¨?$}[{§}X€}"d[£XD!d£!dd¨°!D"[$[$X.&((@@@&@#((.d.(*§.(#@@&%%##£!€$[r{${d¨¨D€€X§?X§°{}€"X.#@#.}°!X"¨{??{d¨Dr§¨[}$}$!$"£D€!°DD[§{"
[§°d}£*&@%.°D?[{°X§°dX$[X€{°§°$£°£[{?D§!°£?rD"dr£?"X°{¨%@@@.Xd£°D}"?[!{£X§°![X£$[°}[°"r¨D!d}$°D§€§£r}£°°!{{XDX¨}$r,@/X,%@@@%(/.¨.(/!,(,#@@/#((§$¨d°"!dDr{}}dD{£€
}{D°¨D€D.##.{§r[€"§€"d€§?!!d§}¨"D?Xdd§}XX{X!$°?§"€€$§{"£r€°€€£!DD{,@/"°![.#@#(.!.((.,(.!.&%,((X?!£$§dd${}€!¨$!§[¨°d£(@#.!€¨§?D{?§$dd£?!£}¨?¨§d£°$X?"X?¨°?°£¨{£$¨
X,#@#{!€![.#&*D°€"€¨rd§°§€X""°D}€Dr$§{X¨?"$¨"!}r£€"D{¨"[r°}€[D°°¨${!?$°dr!"€[£D¨$r[?$"°}["§X§!![§?XX}D°"¨{"r£"£{Xr,@/§[X€{?/@&%€}/(.,(."d.&&#({§€£d{[}"["d¨}}"!{
$d}§"!§€r!¨[£}"¨![§DD$€}€${°§d$}!"}d[r€!£?D€¨$£X{{§dd$¨{!d}d$"X§}X/@,?£?.$d"(@&*[*(*,(.?£r,&#(r[}{£€{XD$£${${!?*#@*£¨}}d[,@@@,°??"[!d§$°}[$[D?r{§XDd£¨€¨[$€$Xr$°
&.Dr€£!}¨,@@&?d$?$§[}{"€X££{d{°!X"X}§§¨¨DDrX¨{}€¨{£r§D!X¨£§°!$[X¨£?D§[€¨X?€[€$!£"§?€°£?r§Dr§§}D£[{?¨r$$"!d£}§£!"XD.,/(//*¨"[.&@%*,(/,(../@@@#([€!D}!??"[€¨$¨X€/@
}d£€D{"{¨¨$°Dd£D?r"{d{°$d}$"!X¨£X°[dXX?!d°$d{}${$![!$?}¨XDd.*(%@@%*€XXd$?¨}D°*@@%.((*#&@@*(@%(d££"§$?}dDX£!}?,@/.!¨${$rdr(@@,r§}}¨°!°°{[X[r[$§!¨¨?D{rd§"§}§€X"£"
XXD{§d}£°&@%°§$€$D$r£r{€X$€$}D[r}$§§°dr!["$¨r§[""§d,(&@&&%,X!d{"€¨"r$}!°D?"{r!£rr$"£r€}[$?£{"¨["Dr}£,(%@@&#*,.°£¨"}[§§"!§°}"D¨&@#!/(#&@,¨.@&((€"[X!{dD[d§!}r{/@,
&@%*.}}!{!¨r}r€X¨[$€$DD!X[Xd£€X$§!r"°$[?€¨$",/#&&&%(*.rX£d¨€{$€X¨$}{rd"£¨¨!€D?%@%*(#@&.}.@@%*(r"!¨"°r?X!"§}{[/@,X{?d"¨r.(@@(¨$"!°£XD[}¨°!€r!§"D°X}§°¨¨r[{?{}"./%
.¨§X"Xd/&@@*drX[{?dX[[{?r?"!£€?°D}°€€?[?}€,(&@%*.D[€$"d$§"rd{[d$$¨¨!?{£d[§£"£¨rX}X§€{.*(##%(*,.D"£§[{D!£!€d°£X£"X£°!}§!!{r$D$D%@%#&@&(.,#@@#*(¨¨d?D$}§°¨!?[{D,@(
?}°rD}§rr}§£§"?£$DX"{!DXX?d.*(%&@@@&#/,..£!{}¨°!rr"D$?[?D}£r¨Dr"dd}{?¨[XD?D$d,@@@&@%,[¨&@*@/*(?$DDD!£"{€$£"X!£*@&[[}€{D!..*.}[r$§DdD"¨££r$!°£$!!"r}$?,#@@&/.!{}d
20 changes: 20 additions & 0 deletions resources/dvc_playground/user/resources/steps/concat_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/usr/bin/env python3
import glob
import logging
from argparse import ArgumentParser
from os.path import isdir

if __name__ == '__main__':
parser = ArgumentParser(description='Concat files from a directory')
parser.add_argument('-i', '--input-dir', required=True, help='Contains files to concat')
parser.add_argument('-o', '--output-file', required=True, help='Result file')

args = parser.parse_args()

if not isdir(args.input_dir):
logging.error(f'Not a directory: {args.input_dir}')
else:
with open(args.output_file, 'w') as fd_write:
for file in sorted(glob.glob(f'{args.input_dir}/*.input')):
with open(file, 'r') as fd_read:
fd_write.write(fd_read.read())
Loading