Remove legacy Jenkins quality pipeline. (#1510)
- it has been used only at Google for a quality pipeline - the worker.py is Python 2 and closely tied to Google Cloud, so I don't believe there is much to be reused by someone else nowadays and not worth the effort to port to Python 3. Signed-off-by: Michael Grupp <grupp@magazino.eu>master
Michael Grupp
GitHub
parent
f30d1f7e1e
commit
051a018c47
|
|
@ -1,122 +0,0 @@
|
|||
# Copyright 2016 The Cartographer Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM ros:kinetic
|
||||
|
||||
ARG CARTOGRAPHER_VERSION=master
|
||||
|
||||
# Xenial's base image doesn't ship with sudo.
|
||||
RUN apt-get update && apt-get install -y sudo time && rm -rf /var/lib/apt/lists/*
|
||||
# First, we invalidate the entire cache if cartographer-project/cartographer has
|
||||
# changed. This file's content changes whenever master changes. See:
|
||||
# http://stackoverflow.com/questions/36996046/how-to-prevent-dockerfile-caching-git-clone
|
||||
ADD https://api.github.com/repos/cartographer-project/cartographer/git/refs/heads/master \
|
||||
cartographer_ros/cartographer_version.json
|
||||
|
||||
# wstool needs the updated rosinstall file to clone the correct repos.
|
||||
COPY cartographer_ros.rosinstall cartographer_ros/
|
||||
COPY scripts/prepare_jenkins_catkin_workspace.sh cartographer_ros/scripts/
|
||||
|
||||
# Invalidates the Docker cache to ensure this command is always executed.
|
||||
ARG CACHEBUST=1
|
||||
RUN CARTOGRAPHER_VERSION=$CARTOGRAPHER_VERSION \
|
||||
cartographer_ros/scripts/prepare_jenkins_catkin_workspace.sh
|
||||
|
||||
# rosdep needs the updated package.xml files to install the correct debs.
|
||||
COPY cartographer_ros/package.xml catkin_ws/src/cartographer_ros/cartographer_ros/
|
||||
COPY cartographer_ros_msgs/package.xml catkin_ws/src/cartographer_ros/cartographer_ros_msgs/
|
||||
COPY cartographer_rviz/package.xml catkin_ws/src/cartographer_ros/cartographer_rviz/
|
||||
COPY scripts/install_debs.sh cartographer_ros/scripts/
|
||||
RUN cartographer_ros/scripts/install_debs.sh && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install proto3.
|
||||
RUN /catkin_ws/src/cartographer/scripts/install_proto3.sh
|
||||
|
||||
# Build, install, and test all packages individually to allow caching. The
|
||||
# ordering of these steps must match the topological package ordering as
|
||||
# determined by Catkin.
|
||||
COPY scripts/install.sh cartographer_ros/scripts/
|
||||
COPY scripts/catkin_test_results.sh cartographer_ros/scripts/
|
||||
|
||||
COPY cartographer_ros_msgs catkin_ws/src/cartographer_ros/cartographer_ros_msgs/
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer_ros_msgs && \
|
||||
cartographer_ros/scripts/install.sh --pkg cartographer_ros_msgs \
|
||||
--catkin-make-args run_tests && \
|
||||
cartographer_ros/scripts/catkin_test_results.sh build_isolated/cartographer_ros_msgs
|
||||
|
||||
RUN cartographer_ros/scripts/install.sh --pkg ceres-solver
|
||||
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer && \
|
||||
cartographer_ros/scripts/install.sh --pkg cartographer --make-args test
|
||||
|
||||
COPY cartographer_ros catkin_ws/src/cartographer_ros/cartographer_ros/
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer_ros && \
|
||||
cartographer_ros/scripts/install.sh --pkg cartographer_ros \
|
||||
--catkin-make-args run_tests && \
|
||||
cartographer_ros/scripts/catkin_test_results.sh build_isolated/cartographer_ros
|
||||
|
||||
COPY cartographer_rviz catkin_ws/src/cartographer_ros/cartographer_rviz/
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer_rviz && \
|
||||
cartographer_ros/scripts/install.sh --pkg cartographer_rviz \
|
||||
--catkin-make-args run_tests && \
|
||||
cartographer_ros/scripts/catkin_test_results.sh build_isolated/cartographer_rviz
|
||||
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer_toru
|
||||
RUN cartographer_ros/scripts/install.sh --pkg cartographer_fetch
|
||||
|
||||
COPY scripts/ros_entrypoint.sh /
|
||||
# A BTRFS bug may prevent us from cleaning up these directories.
|
||||
# https://btrfs.wiki.kernel.org/index.php/Problem_FAQ#I_cannot_delete_an_empty_directory
|
||||
RUN rm -rf cartographer_ros catkin_ws || true
|
||||
|
||||
RUN sudo apt-get update
|
||||
RUN sudo apt-get -y install openjdk-8-jdk python-pip
|
||||
|
||||
ENV HOME /home/jenkins
|
||||
RUN addgroup --system --gid 10000 jenkins
|
||||
RUN adduser --system --ingroup jenkins --home $HOME --uid 10000 jenkins
|
||||
|
||||
LABEL Description="This is a base image, which provides the Jenkins agent executable (slave.jar)" Vendor="Jenkins project" Version="3.17"
|
||||
|
||||
ARG VERSION=3.17
|
||||
ARG AGENT_WORKDIR=/home/jenkins/agent
|
||||
|
||||
RUN curl --create-dirs -sSLo /usr/share/jenkins/slave.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \
|
||||
&& chmod 755 /usr/share/jenkins \
|
||||
&& chmod 644 /usr/share/jenkins/slave.jar
|
||||
# USER jenkins
|
||||
ENV AGENT_WORKDIR=${AGENT_WORKDIR}
|
||||
RUN mkdir /home/jenkins/.jenkins && mkdir -p ${AGENT_WORKDIR}
|
||||
|
||||
VOLUME /home/jenkins/.jenkins
|
||||
VOLUME ${AGENT_WORKDIR}
|
||||
WORKDIR /home/jenkins
|
||||
|
||||
COPY jenkins/jenkins-slave /usr/local/bin/jenkins-slave
|
||||
|
||||
ENV CLOUDSDK_CORE_DISABLE_PROMPTS 1
|
||||
ENV PATH /opt/google-cloud-sdk/bin:$PATH
|
||||
|
||||
USER root
|
||||
|
||||
# Install Google Cloud Components
|
||||
RUN curl https://sdk.cloud.google.com | bash && mv google-cloud-sdk /opt
|
||||
RUN gcloud components install kubectl
|
||||
|
||||
RUN pip install --upgrade google-cloud-datastore
|
||||
RUN pip install --upgrade google-cloud-bigquery
|
||||
COPY jenkins/worker.py /worker.py
|
||||
|
||||
# USER root
|
||||
ENTRYPOINT ["jenkins-slave"]
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
podTemplate(label: 'node-0', containers: [
|
||||
containerTemplate(
|
||||
name: 'jnlp',
|
||||
image: 'eggsy84/gcp-jenkins-slave-k8s-seed:latest',
|
||||
ttyEnabled: false,
|
||||
command: '',
|
||||
privileged: true,
|
||||
alwaysPullImage: false,
|
||||
workingDir: '/home/jenkins',
|
||||
args: '${computer.jnlpmac} ${computer.name}'
|
||||
)
|
||||
],
|
||||
volumes: [
|
||||
secretVolume(mountPath: '/opt/config', secretName: 'gcloud-svc-account'),
|
||||
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
|
||||
persistentVolumeClaim(claimName: 'data-claim-compile', mountPath: '/data'),
|
||||
]
|
||||
) {
|
||||
node('node-0') {
|
||||
stage('Compile') {
|
||||
sh 'gcloud auth activate-service-account --key-file=/opt/config/gcloud-svc-account.json'
|
||||
sh 'cd /data && rm -Rf *'
|
||||
sh 'cd /data && git clone https://github.com/cartographer-project/cartographer_ros'
|
||||
sh 'cd /data/cartographer_ros && docker build -f jenkins/Dockerfile.kinetic -t kinetic-jenkins-slave --build-arg CACHEBUST=$(date +%s) .'
|
||||
}
|
||||
stage('Push') {
|
||||
sh 'docker tag kinetic-jenkins-slave eu.gcr.io/cartographer-141408/kinetic-jenkins-slave'
|
||||
sh 'gcloud docker -- push eu.gcr.io/cartographer-141408/kinetic-jenkins-slave'
|
||||
sh 'cd /data && rm -Rf *'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
podTemplate(label: 'node-1', containers: [
|
||||
containerTemplate(
|
||||
name: 'jnlp',
|
||||
image: 'eu.gcr.io/cartographer-141408/kinetic-jenkins-slave:latest',
|
||||
ttyEnabled: false,
|
||||
command: '',
|
||||
privileged: true,
|
||||
alwaysPullImage: true,
|
||||
workingDir: '/home/jenkins',
|
||||
args: '${computer.jnlpmac} ${computer.name}'
|
||||
)
|
||||
],
|
||||
volumes: [
|
||||
secretVolume(mountPath: '/opt/config', secretName: 'gcloud-svc-account'),
|
||||
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
|
||||
persistentVolumeClaim(claimName: 'data-claim-compile', mountPath: '/data'),
|
||||
]
|
||||
) {
|
||||
node('node-1') {
|
||||
stage('Run Fetch Pipeline') {
|
||||
sh 'gcloud auth activate-service-account --key-file=/opt/config/gcloud-svc-account.json'
|
||||
sh 'GOOGLE_APPLICATION_CREDENTIALS="/opt/config/gcloud-svc-account.json" GOOGLE_CLOUD_DISABLE_GRPC=True python /worker.py --worker_id 0 --num_workers 1 --pipeline_id fetch'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
podTemplate(label: 'node-2', containers: [
|
||||
containerTemplate(
|
||||
name: 'jnlp',
|
||||
image: 'eu.gcr.io/cartographer-141408/kinetic-jenkins-slave:latest',
|
||||
ttyEnabled: false,
|
||||
command: '',
|
||||
privileged: true,
|
||||
alwaysPullImage: true,
|
||||
workingDir: '/home/jenkins',
|
||||
args: '${computer.jnlpmac} ${computer.name}'
|
||||
)
|
||||
],
|
||||
volumes: [
|
||||
secretVolume(mountPath: '/opt/config', secretName: 'gcloud-svc-account'),
|
||||
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
|
||||
persistentVolumeClaim(claimName: 'data-claim-compile', mountPath: '/data'),
|
||||
]
|
||||
) {
|
||||
node('node-2') {
|
||||
stage('Run Backpack Pipeline') {
|
||||
sh 'gcloud auth activate-service-account --key-file=/opt/config/gcloud-svc-account.json'
|
||||
sh 'GOOGLE_APPLICATION_CREDENTIALS="/opt/config/gcloud-svc-account.json" GOOGLE_CLOUD_DISABLE_GRPC=True python /worker.py --worker_id 0 --num_workers 1 --pipeline_id backpack'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
podTemplate(label: 'node-3', containers: [
|
||||
containerTemplate(
|
||||
name: 'jnlp',
|
||||
image: 'eu.gcr.io/cartographer-141408/kinetic-jenkins-slave:latest',
|
||||
ttyEnabled: false,
|
||||
command: '',
|
||||
privileged: true,
|
||||
alwaysPullImage: true,
|
||||
workingDir: '/home/jenkins',
|
||||
args: '${computer.jnlpmac} ${computer.name}'
|
||||
)
|
||||
],
|
||||
volumes: [
|
||||
secretVolume(mountPath: '/opt/config', secretName: 'gcloud-svc-account'),
|
||||
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
|
||||
persistentVolumeClaim(claimName: 'data-claim-compile', mountPath: '/data'),
|
||||
]
|
||||
) {
|
||||
node('node-3') {
|
||||
stage('Run Toru Pipeline') {
|
||||
sh 'gcloud auth activate-service-account --key-file=/opt/config/gcloud-svc-account.json'
|
||||
sh 'GOOGLE_APPLICATION_CREDENTIALS="/opt/config/gcloud-svc-account.json" GOOGLE_CLOUD_DISABLE_GRPC=True python /worker.py --worker_id 0 --num_workers 1 --pipeline_id toru'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
#!/usr/bin/env sh
|
||||
|
||||
# The MIT License
|
||||
#
|
||||
# Copyright (c) 2015, CloudBees, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
# Usage jenkins-slave.sh [options] -url http://jenkins [SECRET] [AGENT_NAME]
|
||||
# Optional environment variables :
|
||||
# * JENKINS_TUNNEL : HOST:PORT for a tunnel to route TCP traffic to jenkins host, when jenkins can't be directly accessed over network
|
||||
# * JENKINS_URL : alternate jenkins URL
|
||||
# * JENKINS_SECRET : agent secret, if not set as an argument
|
||||
# * JENKINS_AGENT_NAME : agent name, if not set as an argument
|
||||
|
||||
if [ $# -eq 1 ]; then
|
||||
|
||||
# if `docker run` only has one arguments, we assume user is running alternate command like `bash` to inspect the image
|
||||
exec "$@"
|
||||
|
||||
else
|
||||
|
||||
# if -tunnel is not provided try env vars
|
||||
case "$@" in
|
||||
*"-tunnel "*) ;;
|
||||
*)
|
||||
if [ ! -z "$JENKINS_TUNNEL" ]; then
|
||||
TUNNEL="-tunnel $JENKINS_TUNNEL"
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
if [ -n "$JENKINS_URL" ]; then
|
||||
URL="-url $JENKINS_URL"
|
||||
fi
|
||||
|
||||
if [ -n "$JENKINS_NAME" ]; then
|
||||
JENKINS_AGENT_NAME="$JENKINS_NAME"
|
||||
fi
|
||||
|
||||
if [ -z "$JNLP_PROTOCOL_OPTS" ]; then
|
||||
echo "Warning: JnlpProtocol3 is disabled by default, use JNLP_PROTOCOL_OPTS to alter the behavior"
|
||||
JNLP_PROTOCOL_OPTS="-Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true"
|
||||
fi
|
||||
|
||||
# If both required options are defined, do not pass the parameters
|
||||
OPT_JENKINS_SECRET=""
|
||||
if [ -n "$JENKINS_SECRET" ]; then
|
||||
case "$@" in
|
||||
*"${JENKINS_SECRET}"*) echo "Warning: SECRET is defined twice in command-line arguments and the environment variable" ;;
|
||||
*)
|
||||
OPT_JENKINS_SECRET="${JENKINS_SECRET}" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
OPT_JENKINS_AGENT_NAME=""
|
||||
if [ -n "$JENKINS_AGENT_NAME" ]; then
|
||||
case "$@" in
|
||||
*"${JENKINS_AGENT_NAME}"*) echo "Warning: AGENT_NAME is defined twice in command-line arguments and the environment variable" ;;
|
||||
*)
|
||||
OPT_JENKINS_AGENT_NAME="${JENKINS_AGENT_NAME}" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
#TODO: Handle the case when the command-line and Environment variable contain different values.
|
||||
#It is fine it blows up for now since it should lead to an error anyway.
|
||||
|
||||
exec java $JAVA_OPTS $JNLP_PROTOCOL_OPTS -cp /usr/share/jenkins/slave.jar hudson.remoting.jnlp.Main -headless $TUNNEL $URL $OPT_JENKINS_SECRET $OPT_JENKINS_AGENT_NAME "$@"
|
||||
fi
|
||||
|
|
@ -1,350 +0,0 @@
|
|||
"""This is the script executed by workers of the quality control pipline."""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
from os.path import basename
|
||||
from pprint import pprint
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from google.cloud import bigquery
|
||||
from google.cloud import datastore
|
||||
|
||||
|
||||
class Pattern(object):
|
||||
"""Defines a pattern for regular expression matching."""
|
||||
|
||||
def __init__(self, pattern):
|
||||
self.regex = re.compile(pattern, re.MULTILINE)
|
||||
|
||||
def extract(self, text):
|
||||
"""Returns a dictionary of named capture groups to extracted output.
|
||||
|
||||
Args:
|
||||
text: input to parse
|
||||
|
||||
Returns an empty dict if no match was found.
|
||||
"""
|
||||
match = self.regex.search(text)
|
||||
if match is None:
|
||||
return {}
|
||||
return match.groupdict()
|
||||
|
||||
def extract_last_occurence(self, text):
|
||||
"""Returns tuple of extracted outputs.
|
||||
|
||||
Args:
|
||||
text: input to parse
|
||||
|
||||
Returns the information extracted from the last match. Returns
|
||||
None if no match was found.
|
||||
"""
|
||||
matches = self.regex.findall(text)
|
||||
if not matches:
|
||||
return None
|
||||
return matches[-1]
|
||||
|
||||
|
||||
# BigQuery table schema
|
||||
SCHEMA = [
|
||||
bigquery.SchemaField('date', 'DATE'),
|
||||
bigquery.SchemaField('commit_sha1', 'STRING'),
|
||||
bigquery.SchemaField('job_id', 'INTEGER'),
|
||||
bigquery.SchemaField('rosbag', 'STRING'),
|
||||
bigquery.SchemaField('user_time_secs', 'FLOAT'),
|
||||
bigquery.SchemaField('system_time_secs', 'FLOAT'),
|
||||
bigquery.SchemaField('wall_time_secs', 'FLOAT'),
|
||||
bigquery.SchemaField('max_set_size_kbytes', 'INTEGER'),
|
||||
bigquery.SchemaField('constraints_count', 'INTEGER'),
|
||||
bigquery.SchemaField('constraints_score_minimum', 'FLOAT'),
|
||||
bigquery.SchemaField('constraints_score_maximum', 'FLOAT'),
|
||||
bigquery.SchemaField('constraints_score_mean', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_abs_trans_err', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_abs_trans_err_dev', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_sqr_trans_err', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_sqr_trans_err_dev', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_abs_rot_err', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_abs_rot_err_dev', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_sqr_rot_err', 'FLOAT'),
|
||||
bigquery.SchemaField('ground_truth_sqr_rot_err_dev', 'FLOAT')
|
||||
]
|
||||
|
||||
# Pattern matchers for the various fields of the '/usr/bin/time -v' output
|
||||
USER_TIME_PATTERN = Pattern(
|
||||
r'^\s*User time \(seconds\): (?P<user_time>\d+.\d+|\d+)')
|
||||
SYSTEM_TIME_PATTERN = Pattern(
|
||||
r'^\s*System time \(seconds\): (?P<system_time>\d+.\d+|\d+)')
|
||||
WALL_TIME_PATTERN = Pattern(
|
||||
r'^\s*Elapsed \(wall clock\) time \(h:mm:ss or m:ss\): '
|
||||
r'((?P<hours>\d{1,2}):|)(?P<minutes>\d{1,2}):(?P<seconds>\d{2}\.\d{2})')
|
||||
MAX_RES_SET_SIZE_PATTERN = Pattern(
|
||||
r'^\s*Maximum resident set size \(kbytes\): (?P<max_set_size>\d+)')
|
||||
CONSTRAINT_STATS_PATTERN = Pattern(
|
||||
r'Score histogram:[\n\r]+'
|
||||
r'Count:\s+(?P<constraints_count>\d+)\s+'
|
||||
r'Min:\s+(?P<constraints_score_min>\d+\.\d+)\s+'
|
||||
r'Max:\s+(?P<constraints_score_max>\d+\.\d+)\s+'
|
||||
r'Mean:\s+(?P<constraints_score_mean>\d+\.\d+)')
|
||||
GROUND_TRUTH_STATS_PATTERN = Pattern(
|
||||
r'Result:[\n\r]+'
|
||||
r'Abs translational error (?P<abs_trans_err>\d+\.\d+) '
|
||||
r'\+/- (?P<abs_trans_err_dev>\d+\.\d+) m[\n\r]+'
|
||||
r'Sqr translational error (?P<sqr_trans_err>\d+\.\d+) '
|
||||
r'\+/- (?P<sqr_trans_err_dev>\d+\.\d+) m\^2[\n\r]+'
|
||||
r'Abs rotational error (?P<abs_rot_err>\d+\.\d+) '
|
||||
r'\+/- (?P<abs_rot_err_dev>\d+\.\d+) deg[\n\r]+'
|
||||
r'Sqr rotational error (?P<sqr_rot_err>\d+\.\d+) '
|
||||
r'\+/- (?P<sqr_rot_err_dev>\d+\.\d+) deg\^2')
|
||||
|
||||
# Pattern matcher for extracting the HEAD commit SHA-1 hash.
|
||||
GIT_SHA1_PATTERN = Pattern(r'^(?P<sha1>[0-9a-f]{40})\s+HEAD')
|
||||
|
||||
|
||||
def get_head_git_sha1():
|
||||
"""Returns the SHA-1 hash of the commit tagged HEAD."""
|
||||
output = subprocess.check_output([
|
||||
'git', 'ls-remote',
|
||||
'https://github.com/cartographer-project/cartographer.git'
|
||||
])
|
||||
parsed = GIT_SHA1_PATTERN.extract(output)
|
||||
return parsed['sha1']
|
||||
|
||||
|
||||
def extract_stats(inp):
|
||||
"""Returns a dictionary of stats."""
|
||||
result = {}
|
||||
|
||||
parsed = USER_TIME_PATTERN.extract(inp)
|
||||
result['user_time_secs'] = float(parsed['user_time'])
|
||||
|
||||
parsed = SYSTEM_TIME_PATTERN.extract(inp)
|
||||
result['system_time_secs'] = float(parsed['system_time'])
|
||||
|
||||
parsed = WALL_TIME_PATTERN.extract(inp)
|
||||
result['wall_time_secs'] = float(parsed['hours'] or 0.) * 3600 + float(
|
||||
parsed['minutes']) * 60 + float(parsed['seconds'])
|
||||
|
||||
parsed = MAX_RES_SET_SIZE_PATTERN.extract(inp)
|
||||
result['max_set_size_kbytes'] = int(parsed['max_set_size'])
|
||||
|
||||
parsed = CONSTRAINT_STATS_PATTERN.extract_last_occurence(inp)
|
||||
print parsed
|
||||
result['constraints_count'] = int(parsed[0])
|
||||
result['constraints_score_min'] = float(parsed[1])
|
||||
result['constraints_score_max'] = float(parsed[2])
|
||||
result['constraints_score_mean'] = float(parsed[3])
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_ground_truth_stats(input_text):
|
||||
"""Returns a dictionary of ground truth stats."""
|
||||
result = {}
|
||||
parsed = GROUND_TRUTH_STATS_PATTERN.extract(input_text)
|
||||
for name in ('abs_trans_err', 'sqr_trans_err', 'abs_rot_err', 'sqr_rot_err'):
|
||||
result['ground_truth_{}'.format(name)] = float(parsed[name])
|
||||
result['ground_truth_{}_dev'.format(name)] = float(
|
||||
parsed['{}_dev'.format(name)])
|
||||
return result
|
||||
|
||||
|
||||
def retrieve_entity(datastore_client, kind, identifier):
|
||||
"""Convenience function for Datastore entity retrieval."""
|
||||
key = datastore_client.key(kind, identifier)
|
||||
return datastore_client.get(key)
|
||||
|
||||
|
||||
def create_job_selector(worker_id, num_workers):
|
||||
"""Constructs a round-robin job selector."""
|
||||
return lambda job_id: job_id % num_workers == worker_id
|
||||
|
||||
|
||||
def run_cmd(cmd):
|
||||
"""Runs command both printing its stdout output and returning it as string."""
|
||||
print cmd
|
||||
p = subprocess.Popen(
|
||||
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
run_cmd.output = []
|
||||
|
||||
def process(line):
|
||||
run_cmd.output.append(line)
|
||||
print line.rstrip()
|
||||
|
||||
while p.poll() is None:
|
||||
process(p.stdout.readline())
|
||||
process(p.stdout.read())
|
||||
return '\n'.join(run_cmd.output)
|
||||
|
||||
|
||||
def run_ros_cmd(ros_distro, ros_cmd):
|
||||
"""Runs command similar to run_cmd but sets ROS environment variables."""
|
||||
cmd = ('/bin/bash -c \"source /opt/ros/{}/setup.bash && source '
|
||||
'/opt/cartographer_ros/setup.bash && {}\"').format(
|
||||
ros_distro, ros_cmd)
|
||||
return run_cmd(cmd)
|
||||
|
||||
|
||||
class Job(object):
|
||||
"""Represents a single job to be executed.
|
||||
|
||||
A job consists of a combination of rosbag and configuration and launch files.
|
||||
"""
|
||||
|
||||
def __init__(self, datastore_client, job_id):
|
||||
self.id = job_id
|
||||
entity = retrieve_entity(datastore_client, 'Job', job_id)
|
||||
self.launch_file = entity['launch_file']
|
||||
self.assets_writer_launch_file = entity['assets_writer_launch_file']
|
||||
self.assets_writer_config_file = entity['assets_writer_config_file']
|
||||
self.rosbag = entity['rosbag']
|
||||
self.ros_package = entity['ros_package']
|
||||
|
||||
def __repr__(self):
|
||||
return 'Job: id : {} launch_file: {} rosbag: {}'.format(
|
||||
self.id, self.launch_file, self.rosbag)
|
||||
|
||||
def run(self, ros_distro, run_id):
|
||||
"""Runs the job with ROS distro 'ros_distro'."""
|
||||
print 'running job {}'.format(self.id)
|
||||
|
||||
# Garbage collect any left-overs from previous runs.
|
||||
run_cmd('rm -rf /data/*')
|
||||
|
||||
# Copy the rosbag to scratch space
|
||||
scratch_dir = '/data/{}'.format(self.id)
|
||||
rosbag_filename = basename(self.rosbag)
|
||||
run_cmd('mkdir {}'.format(scratch_dir))
|
||||
run_cmd('gsutil cp gs://{} {}/{}'.format(self.rosbag, scratch_dir,
|
||||
rosbag_filename))
|
||||
|
||||
# Creates pbstream
|
||||
output = run_ros_cmd(ros_distro,
|
||||
'/usr/bin/time -v roslaunch {} {} '
|
||||
'bag_filenames:={}/{} no_rviz:=true'.format(
|
||||
self.ros_package, self.launch_file, scratch_dir,
|
||||
rosbag_filename))
|
||||
info = extract_stats(output)
|
||||
|
||||
# Creates assets.
|
||||
run_ros_cmd(
|
||||
ros_distro, '/usr/bin/time -v roslaunch {} {} '
|
||||
'bag_filenames:={}/{} pose_graph_filename:='
|
||||
'{}/{}.pbstream config_file:={}'.format(
|
||||
self.ros_package, self.assets_writer_launch_file, scratch_dir,
|
||||
rosbag_filename, scratch_dir, rosbag_filename,
|
||||
self.assets_writer_config_file))
|
||||
|
||||
# Copies assets to bucket.
|
||||
run_cmd('gsutil cp {}/{}.pbstream '
|
||||
'gs://cartographer-ci-artifacts/{}/{}/{}.pbstream'.format(
|
||||
scratch_dir, rosbag_filename, run_id, self.id, rosbag_filename))
|
||||
run_cmd('gsutil cp {}/{}_* gs://cartographer-ci-artifacts/{}/{}/'.format(
|
||||
scratch_dir, rosbag_filename, run_id, self.id))
|
||||
|
||||
# Download ground truth relations file.
|
||||
run_cmd('gsutil cp gs://cartographer-ci-ground-truth/{}/relations.pb '
|
||||
'{}/relations.pb'.format(self.id, scratch_dir))
|
||||
|
||||
# Calculate metrics.
|
||||
output = run_ros_cmd(ros_distro, 'cartographer_compute_relations_metrics '
|
||||
'-relations_filename {}/relations.pb '
|
||||
'-pose_graph_filename {}/{}.pbstream'.format(
|
||||
scratch_dir, scratch_dir, rosbag_filename))
|
||||
|
||||
# Add ground truth stats.
|
||||
info.update(extract_ground_truth_stats(output))
|
||||
|
||||
info['rosbag'] = rosbag_filename
|
||||
return info
|
||||
|
||||
|
||||
class Worker(object):
|
||||
"""Represents a single worker that executes a sequence of Jobs."""
|
||||
|
||||
def __init__(self, datastore_client, pipeline_id, run_id):
|
||||
entity = retrieve_entity(datastore_client, 'PipelineConfig', pipeline_id)
|
||||
self.pipeline_id = pipeline_id
|
||||
self.jobs = [Job(datastore_client, job_id) for job_id in entity['jobs']]
|
||||
self.scratch_dir = entity['scratch_dir']
|
||||
self.ros_distro = entity['ros_distro']
|
||||
self.run_id = run_id
|
||||
|
||||
def __repr__(self):
|
||||
result = 'Worker: pipeline_id: {}\n'.format(self.pipeline_id)
|
||||
for job in self.jobs:
|
||||
result += '{}\n'.format(str(job))
|
||||
return result
|
||||
|
||||
def run_jobs(self, selector):
|
||||
outputs = {}
|
||||
for idx, job in enumerate(self.jobs):
|
||||
if selector(idx):
|
||||
output = job.run(self.ros_distro, self.run_id)
|
||||
outputs[job.id] = output
|
||||
else:
|
||||
print 'job {}: skip'.format(job.id)
|
||||
return outputs
|
||||
|
||||
|
||||
def publish_stats_to_big_query(stats_dict, now, head_sha1):
|
||||
"""Publishes metrics to BigQuery."""
|
||||
bigquery_client = bigquery.Client()
|
||||
dataset = bigquery_client.dataset('Cartographer')
|
||||
table = dataset.table('metrics')
|
||||
rows_to_insert = []
|
||||
for job_identifier, job_info in stats_dict.iteritems():
|
||||
print job_info
|
||||
data = ('{}-{}-{}'.format(
|
||||
now.year, now.month,
|
||||
now.day), head_sha1, job_identifier, job_info['rosbag'],
|
||||
job_info['user_time_secs'], job_info['system_time_secs'],
|
||||
job_info['wall_time_secs'], job_info['max_set_size_kbytes'],
|
||||
job_info['constraints_count'], job_info['constraints_score_min'],
|
||||
job_info['constraints_score_max'],
|
||||
job_info['constraints_score_mean'],
|
||||
job_info['ground_truth_abs_trans_err'],
|
||||
job_info['ground_truth_abs_trans_err_dev'],
|
||||
job_info['ground_truth_sqr_trans_err'],
|
||||
job_info['ground_truth_sqr_trans_err_dev'],
|
||||
job_info['ground_truth_abs_rot_err'],
|
||||
job_info['ground_truth_abs_rot_err_dev'],
|
||||
job_info['ground_truth_sqr_rot_err'],
|
||||
job_info['ground_truth_sqr_rot_err_dev'])
|
||||
|
||||
rows_to_insert.append(data)
|
||||
|
||||
errors = bigquery_client.create_rows(
|
||||
table, rows_to_insert, selected_fields=SCHEMA)
|
||||
if not errors:
|
||||
print 'Pushed {} row(s) into Cartographer:metrics'.format(
|
||||
len(rows_to_insert))
|
||||
else:
|
||||
print 'Errors:'
|
||||
pprint(errors)
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""Parses the command line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--worker_id', type=int)
|
||||
parser.add_argument('--num_workers', type=int)
|
||||
parser.add_argument('--pipeline_id', type=str)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
ds_client = datastore.Client()
|
||||
job_selector = create_job_selector(int(args.worker_id), int(args.num_workers))
|
||||
head_sha1 = get_head_git_sha1()
|
||||
now = datetime.datetime.now()
|
||||
pipeline_run_id = '{}-{}-{}_{}'.format(now.year, now.month, now.day,
|
||||
head_sha1)
|
||||
worker = Worker(ds_client, args.pipeline_id, pipeline_run_id)
|
||||
stats_dict = worker.run_jobs(job_selector)
|
||||
publish_stats_to_big_query(stats_dict, now, head_sha1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue