move to github container registry + CI speedup (commaai#19621)

gaanthony · Dec 31, 2020 · 59d7c5e · 59d7c5e
1 parent 309384e
commit 59d7c5e
Show file tree

Hide file tree

Showing 11 changed files with 218 additions and 227 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
diff --git a/Dockerfile.openpilotci → Dockerfile.openpilot b/Dockerfile.openpilotci → Dockerfile.openpilot
@@ -23,3 +23,6 @@ COPY ./opendbc /tmp/openpilot/opendbc
 COPY ./cereal /tmp/openpilot/cereal
 COPY ./panda /tmp/openpilot/panda
 COPY ./selfdrive /tmp/openpilot/selfdrive
+
+WORKDIR /tmp/openpilot
+RUN scons -j$(nproc)
diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base
@@ -60,4 +60,5 @@ RUN pyenv install 3.8.2 && \
     pip install --no-cache-dir pipenv==2020.8.13 && \
     cd /tmp && \
     pipenv install --system --deploy --dev --clear && \
+    pip uninstall torch tensorflow -y && \
     pip uninstall -y pipenv
diff --git a/selfdrive/loggerd/tests/loggerd_tests_common.py b/selfdrive/loggerd/tests/loggerd_tests_common.py
@@ -76,6 +76,7 @@ def setUp(self):
     uploader.Params = MockParams
     uploader.fake_upload = True
     uploader.force_wifi = True
+    uploader.allow_sleep = False
     self.seg_num = random.randint(1, 300)
     self.seg_format = "2019-04-18--12-52-54--{}"
     self.seg_format2 = "2019-05-18--11-22-33--{}"

diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py
@@ -19,6 +19,7 @@
 UPLOAD_ATTR_NAME = 'user.upload'
 UPLOAD_ATTR_VALUE = b'1'
 
+allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
 force_wifi = os.getenv("FORCEWIFI") is not None
 fake_upload = os.getenv("FAKEUPLOAD") is not None
 
@@ -208,7 +209,8 @@ def uploader_fn(exit_event):
 
     d = uploader.next_file_to_upload(with_raw=allow_raw_upload and on_wifi and offroad)
     if d is None:  # Nothing to upload
-      time.sleep(60 if offroad else 5)
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
       continue
 
     key, fn = d
@@ -218,7 +220,7 @@ def uploader_fn(exit_event):
     success = uploader.upload(key, fn)
     if success:
       backoff = 0.1
-    else:
+    elif allow_sleep:
       cloudlog.info("backoff %r", backoff)
       time.sleep(backoff + random.uniform(0, backoff))
       backoff = min(backoff*2, 120)

diff --git a/selfdrive/test/longitudinal_maneuvers/test_longitudinal.py b/selfdrive/test/longitudinal_maneuvers/test_longitudinal.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import os
-os.environ['OLD_CAN'] = '1'
 os.environ['NOCRASH'] = '1'
 
 import unittest
@@ -11,16 +10,10 @@
 from selfdrive.car.honda.values import CruiseButtons as CB
 from selfdrive.test.longitudinal_maneuvers.maneuver import Maneuver
 import selfdrive.manager as manager
+from common.file_helpers import mkdirs_exists_ok
 from common.params import Params
 
 
-def create_dir(path):
-  try:
-    os.makedirs(path)
-  except OSError:
-    pass
-
-
 def check_no_collision(log):
   return min(log['d_rel']) > 0
 
@@ -317,7 +310,7 @@ def setup_output():
         view_html += "<td><img class='maneuver_graph' src='%s'/></td>" % (os.path.join("maneuver" + str(i + 1).zfill(2), c), )
       view_html += "</tr>"
 
-    create_dir(output_dir)
+    mkdirs_exists_ok(output_dir)
     with open(os.path.join(output_dir, "index.html"), "w") as f:
       f.write(view_html)
 

diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py
@@ -346,7 +346,7 @@ def python_replay_process(cfg, lr):
         os.environ['FINGERPRINT'] = msg.carParams.carFingerprint
       break
 
-  manager.prepare_managed_process(cfg.proc_name)
+  manager.prepare_managed_process(cfg.proc_name, build=True)
   mod = importlib.import_module(manager.managed_processes[cfg.proc_name])
   thread = threading.Thread(target=mod.main, args=args)
   thread.daemon = True

diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py
@@ -14,13 +14,12 @@
 segments = [
   ("HONDA", "d83f36766f8012a5|2020-02-05--18-42-21--2"),      # HONDA.CIVIC_BOSCH_DIESEL (BOSCH)
   ("HONDA", "a74b011b32b51b56|2020-07-26--17-09-36--6"),      # HONDA.CIVIC (NIDEC)
-  ("TOYOTA", "77611a1fac303767|2020-02-29--13-29-33--3"),     # TOYOTA.COROLLA_TSS2
   ("TOYOTA", "b14c5b4742e6fc85|2020-10-14--11-04-47--4"),     # TOYOTA.RAV4  (LQR)
   ("TOYOTA", "0982d79ebb0de295|2020-10-18--19-11-36--5"),     # TOYOTA.PRIUS (INDI)
   ("CHRYSLER", "b6849f5cf2c926b1|2020-02-28--07-29-48--13"),  # CHRYSLER.PACIFICA
   ("HYUNDAI", "5b7c365c50084530|2020-04-15--16-13-24--3"),    # HYUNDAI.SONATA
-  ("SUBARU", "c321c6b697c5a5ff|2020-06-23--11-04-33--12"),     # SUBARU.FORESTER
-  ("VOLKSWAGEN", "76b83eb0245de90e|2020-03-05--19-16-05--3"),  # VW.GOLF
+  ("SUBARU", "c321c6b697c5a5ff|2020-06-23--11-04-33--12"),    # SUBARU.FORESTER
+  ("VOLKSWAGEN", "76b83eb0245de90e|2020-03-05--19-16-05--3"), # VW.GOLF
   ("NISSAN", "fbbfa6af821552b9|2020-03-03--08-09-43--0"),     # NISSAN.XTRAIL
 
   # TODO: update this route
@@ -170,9 +169,11 @@ def format_diff(results, ref_commit):
     f.write(diff2)
   print(diff1)
 
-  print("TEST", "FAILED" if failed else "SUCCEEDED")
-
-  print("\n\nTo update the reference logs for this test run:")
-  print("./update_refs.py")
+  if failed:
+    print("TEST FAILED")
+    print("\n\nTo update the reference logs for this test run:")
+    print("./update_refs.py")
+  else:
+    print("TEST SUCCEEDED")
 
   sys.exit(int(failed))
diff --git a/tools/lib/tests/test_caching.py b/tools/lib/tests/test_caching.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
-
 import os
-import unittest
 import shutil
+import unittest
+
+os.environ["COMMA_CACHE"] = "/tmp/__test_cache__"
 from tools.lib.url_file import URLFile, CACHE_DIR
 
 

diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py
@@ -14,7 +14,7 @@
 K = 1000
 CHUNK_SIZE = 1000 * K
 
-CACHE_DIR = "/tmp/comma_download_cache/"
+CACHE_DIR = os.environ.get("COMMA_CACHE", "/tmp/comma_download_cache/")
 
 
 def hash_256(link):

diff --git a/tools/webcam/Dockerfile b/tools/webcam/Dockerfile
@@ -4,24 +4,40 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
 
 # install opencv
-RUN export OPENCV_VERSION='4.2.0' && \
-    sudo apt-get -y update && \
-    sudo apt-get install -y build-essential cmake && \
-    sudo apt-get install -y qt5-default libvtk6-dev && \
-    sudo apt-get install -y libdc1394-22-dev libavcodec-dev libavformat-dev libswscale-dev \
-                            libtheora-dev libvorbis-dev libxvidcore-dev libx264-dev yasm \
-                            libopencore-amrnb-dev libopencore-amrwb-dev libv4l-dev libxine2-dev && \
-    sudo apt-get install -y libtbb-dev libeigen3-dev && \
+ENV OPENCV_VERSION '4.2.0'
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      libvtk6-dev \
+      libdc1394-22-dev \
+      libavcodec-dev \
+      libavformat-dev \
+      libswscale-dev \
+      libtheora-dev \
+      libvorbis-dev \
+      libxvidcore-dev \
+      libx264-dev \
+      yasm \
+      libopencore-amrnb-dev \
+      libopencore-amrwb-dev \
+      libv4l-dev \
+      libxine2-dev \
+      libtbb-dev \
+    && rm -rf /var/lib/apt/lists/* && \
+
+    mkdir /tmp/opencv_build && \
+    cd /tmp/opencv_build && \
 
     wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz && \
-    tar -xvf ${OPENCV_VERSION}.tar.gz && rm ${OPENCV_VERSION}.tar.gz && \
+    tar -xvf ${OPENCV_VERSION}.tar.gz && \
     mv opencv-${OPENCV_VERSION} OpenCV && \
     cd OpenCV && mkdir build && cd build && \
     cmake -DWITH_OPENGL=ON -DFORCE_VTK=ON -DWITH_TBB=ON -DWITH_GDAL=ON \
           -DWITH_XINE=ON -DENABLE_PRECOMPILED_HEADERS=OFF .. && \
     make -j8 && \
-    sudo make install && \
-    sudo ldconfig
+    make install && \
+    ldconfig && \
+
+    cd /tmp && rm -rf /tmp/opencv_build
 
 
 RUN mkdir -p /tmp/openpilot