Skip to content

Commit d8c01aa

Browse files
vjrjvjrj
authored
Added two pipelines jobs to copy/delete specific drs to hdfs (#590)
1 parent 6acdecd commit d8c01aa

File tree

2 files changed

+147
-0
lines changed

2 files changed

+147
-0
lines changed

‎ansible/roles/pipelines_jenkins/templates/jenkins/jobs/Copy-DwCA/config.xml.j2

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?xml version='1.1' encoding='UTF-8'?>
2+
<project>
3+
<actions/>
4+
<description>Download a DwCA from collectory and copy it to HDFS</description>
5+
<keepDependencies>false</keepDependencies>
6+
<properties>
7+
<com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="gitlab-plugin@1.5.33">
8+
<gitLabConnection></gitLabConnection>
9+
<jobCredentialId></jobCredentialId>
10+
<useAlternativeCredential>false</useAlternativeCredential>
11+
</com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty>
12+
<org.jenkins.ci.plugins.html5__notifier.JobPropertyImpl plugin="html5-notifier-plugin@1.5">
13+
<skip>false</skip>
14+
</org.jenkins.ci.plugins.html5__notifier.JobPropertyImpl>
15+
<com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.34">
16+
<autoRebuild>false</autoRebuild>
17+
<rebuildDisabled>false</rebuildDisabled>
18+
</com.sonyericsson.rebuild.RebuildSettings>
19+
<hudson.model.ParametersDefinitionProperty>
20+
<parameterDefinitions>
21+
<hudson.model.StringParameterDefinition>
22+
<name>datasetId</name>
23+
<description>The datasetId to be copied (space separated) e.g. dr893</description>
24+
<trim>false</trim>
25+
</hudson.model.StringParameterDefinition>
26+
</parameterDefinitions>
27+
</hudson.model.ParametersDefinitionProperty>
28+
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.8">
29+
<maxConcurrentPerNode>0</maxConcurrentPerNode>
30+
<maxConcurrentTotal>0</maxConcurrentTotal>
31+
<categories class="java.util.concurrent.CopyOnWriteArrayList"/>
32+
<throttleEnabled>false</throttleEnabled>
33+
<throttleOption>project</throttleOption>
34+
<limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
35+
<paramsToUseForLimit></paramsToUseForLimit>
36+
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>
37+
</properties>
38+
<scm class="hudson.scm.NullSCM"/>
39+
<assignedNode>{{ master_node }}</assignedNode>
40+
<canRoam>false</canRoam>
41+
<disabled>false</disabled>
42+
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
43+
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
44+
<triggers/>
45+
<concurrentBuild>false</concurrentBuild>
46+
<builders>
47+
<hudson.tasks.Shell>
48+
<command>echo &quot;$datasetId&quot; &gt; setbuildname.txt
49+
</command>
50+
<configuredLocalRules/>
51+
</hudson.tasks.Shell>
52+
<org.jenkinsci.plugins.buildnameupdater.BuildNameUpdater plugin="build-name-setter@2.2.0">
53+
<buildName>setbuildname.txt</buildName>
54+
<macroTemplate>#${BUILD_NUMBER}</macroTemplate>
55+
<fromFile>true</fromFile>
56+
<fromMacro>false</fromMacro>
57+
<macroFirst>false</macroFirst>
58+
</org.jenkinsci.plugins.buildnameupdater.BuildNameUpdater>
59+
<hudson.tasks.Shell>
60+
<command>sudo -u spark la-pipelines copy $datasetId --hdfs
61+
</command>
62+
<configuredLocalRules/>
63+
</hudson.tasks.Shell>
64+
</builders>
65+
<publishers/>
66+
<buildWrappers>
67+
<hudson.plugins.ansicolor.AnsiColorBuildWrapper plugin="ansicolor@1.0.1">
68+
<colorMapName>xterm</colorMapName>
69+
</hudson.plugins.ansicolor.AnsiColorBuildWrapper>
70+
</buildWrappers>
71+
</project>

‎ansible/roles/pipelines_jenkins/templates/jenkins/jobs/Delete-dwca-from-hdfs/config.xml.j2

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<?xml version='1.1' encoding='UTF-8'?>
2+
<project>
3+
<actions/>
4+
<description>Delete a DwCA</description>
5+
<keepDependencies>false</keepDependencies>
6+
<properties>
7+
<com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty plugin="gitlab-plugin@1.5.33">
8+
<gitLabConnection></gitLabConnection>
9+
<jobCredentialId></jobCredentialId>
10+
<useAlternativeCredential>false</useAlternativeCredential>
11+
</com.dabsquared.gitlabjenkins.connection.GitLabConnectionProperty>
12+
<org.jenkins.ci.plugins.html5__notifier.JobPropertyImpl plugin="html5-notifier-plugin@1.5">
13+
<skip>false</skip>
14+
</org.jenkins.ci.plugins.html5__notifier.JobPropertyImpl>
15+
<com.sonyericsson.rebuild.RebuildSettings plugin="rebuild@1.34">
16+
<autoRebuild>false</autoRebuild>
17+
<rebuildDisabled>false</rebuildDisabled>
18+
</com.sonyericsson.rebuild.RebuildSettings>
19+
<hudson.model.ParametersDefinitionProperty>
20+
<parameterDefinitions>
21+
<hudson.model.StringParameterDefinition>
22+
<name>datasetId</name>
23+
<description>The datasetId to be delete (space separated) e.g. dr893</description>
24+
<trim>false</trim>
25+
</hudson.model.StringParameterDefinition>
26+
</parameterDefinitions>
27+
</hudson.model.ParametersDefinitionProperty>
28+
<hudson.plugins.throttleconcurrents.ThrottleJobProperty plugin="throttle-concurrents@2.8">
29+
<maxConcurrentPerNode>0</maxConcurrentPerNode>
30+
<maxConcurrentTotal>0</maxConcurrentTotal>
31+
<categories class="java.util.concurrent.CopyOnWriteArrayList"/>
32+
<throttleEnabled>false</throttleEnabled>
33+
<throttleOption>project</throttleOption>
34+
<limitOneJobWithMatchingParams>false</limitOneJobWithMatchingParams>
35+
<paramsToUseForLimit></paramsToUseForLimit>
36+
</hudson.plugins.throttleconcurrents.ThrottleJobProperty>
37+
</properties>
38+
<scm class="hudson.scm.NullSCM"/>
39+
<assignedNode>{{ master_node }}</assignedNode>
40+
<canRoam>false</canRoam>
41+
<disabled>false</disabled>
42+
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
43+
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
44+
<triggers/>
45+
<concurrentBuild>false</concurrentBuild>
46+
<builders>
47+
<hudson.tasks.Shell>
48+
<command>echo &quot;$datasetId&quot; &gt; setbuildname.txt
49+
</command>
50+
<configuredLocalRules/>
51+
</hudson.tasks.Shell>
52+
<org.jenkinsci.plugins.buildnameupdater.BuildNameUpdater plugin="build-name-setter@2.2.0">
53+
<buildName>setbuildname.txt</buildName>
54+
<macroTemplate>#${BUILD_NUMBER}</macroTemplate>
55+
<fromFile>true</fromFile>
56+
<fromMacro>false</fromMacro>
57+
<macroFirst>false</macroFirst>
58+
</org.jenkinsci.plugins.buildnameupdater.BuildNameUpdater>
59+
<hudson.tasks.Shell>
60+
<command>#!/bin/bash
61+
for f in $datasetId
62+
do
63+
echo &quot;Remove $f from DFS&quot;
64+
sudo -u spark /data/hadoop/bin/hdfs dfs -rm -r -f /dwca-exports/$f
65+
done
66+
</command>
67+
<configuredLocalRules/>
68+
</hudson.tasks.Shell>
69+
</builders>
70+
<publishers/>
71+
<buildWrappers>
72+
<hudson.plugins.ansicolor.AnsiColorBuildWrapper plugin="ansicolor@1.0.1">
73+
<colorMapName>xterm</colorMapName>
74+
</hudson.plugins.ansicolor.AnsiColorBuildWrapper>
75+
</buildWrappers>
76+
</project>

0 commit comments

Comments
 (0)