Clang-10 and actually fix dev image to be Ubuntu 20.04.

Summary:
This fixes the following issue related to libtinfo on PEM/Kelvin start:
`libtinfo.so.5: cannot open shared object file: No such file or directory`

When we updated to 20.04, I think I changed the base image but not the active dev image.
This means that we were building all the images using the old ubuntu 18.04.
James noticed a break with skaffold that caused the PEMs to break because of relying on deprecated terminfo5 so he changed our sid deploy dependency to terminfo6. Unfortunately, this broke our deploy images.

I updated the dev images to be ubuntu 18.04, but sadly Clang on that relies on terminfo5 so it's broken with ubuntu 20.04. It's fixable by installing terminfo5, but this might break our code if it links to v5 instead of v6, since we only ship v6.

Updating the old clang install didn't work. clang-9 does not compile with clang-10. Ubuntu 20.04 ships with clang-10 so the compiler can't build our old clang-9 deb with terminfo6...

This updates the compiler to clang-10 with all the related changes necessary to make sure the code still compiles.

Test Plan:
`bazel test //...` in the docker VM.
`bazel run src/vizier/services/agent:pem_image` in the docker VM.

Reviewers: oazizi, michelle, jamesbartlett, philkuz, #engineering

Reviewed By: oazizi, #engineering

Differential Revision: https://phab.corp.pixielabs.ai/D5623

GitOrigin-RevId: a26680e32211f9aec5e42c1349c4cd35015e2213

.bazelrc

 # Global bazelrc file, see https://docs.bazel.build/versions/master/guide.html#bazelrc.
 
 # This is to flush bazel cache for new version of clang/gcc.
-build --action_env=CLANG_COMPILER_VERSION=9.0_2
+build --action_env=CLANG_COMPILER_VERSION=10.0_1
 
 # Get version information into the build.
 build --workspace_status_command=bazel/get_workspace_status
 
-# Some of our dependencies still need Python 2.x.
-build --host_force_python=PY2
+# We only have support for python3 on the Ubuntu 20.04 image.
+build --host_force_python=PY3
 
 # Limit memory for JVM to reduce runaway usage.
 startup --host_jvm_args=-Xmx1g
@@ -44,8 +44,8 @@ build:libc++ --action_env=CXXFLAGS=-stdlib=libc++
 build:libc++ --action_env=LDFLAGS=-stdlib=libc++
 build:libc++ --action_env=BAZEL_CXXOPTS=-std=c++17:-stdlib=libc++
 build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm
-build:libc++ --action_env=CC=/opt/clang-9.0-libc++/bin/clang
-build:libc++ --action_env=CXX=/opt/clang-9.0-libc++/bin/clang++
+build:libc++ --action_env=CC=/opt/clang-10.0-libc++/bin/clang
+build:libc++ --action_env=CXX=/opt/clang-10.0-libc++/bin/clang++
 build:libc++ --define use_libcpp=1
 # TODO(zasgar): Remove this in next diff.
 build:libc++ --copt -D_LIBCPP_NO_EXPERIMENTAL_DEPRECATION_WARNING_FILESYSTEM
@@ -87,8 +87,8 @@ build:asan --copt -DADDRESS_SANITIZER=1
 build:asan --test_env=ASAN_SYMBOLIZER_PATH
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true
 build:asan --define tcmalloc=disabled
-# Clang-9 requires additional linkopts to make sure ubsan is linked in correctly.
-build:asan --linkopt=-L/opt/clang-9.0/lib/clang/9.0.0/lib/linux
+# Clang-10 requires additional linkopts to make sure ubsan is linked in correctly.
+build:asan --linkopt=-L/opt/clang-10.0/lib/clang/10.0.0/lib/linux
 build:asan --linkopt=-l:libclang_rt.ubsan_standalone-x86_64.a
 build:asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx-x86_64.a
 

.gitmodules

@@ -55,3 +55,6 @@
 [submodule "third_party/kuberesolver"]
 	path = third_party/kuberesolver
 	url = https://github.com/pixie-labs/kuberesolver
+[submodule "chef/site-cookbooks/alternatives"]
+	path = chef/site-cookbooks/alternatives
+	url = https://github.com/vkhatri/chef-alternatives

.vscode/c_cpp_properties.json

@@ -9,9 +9,9 @@
             "macFrameworkPath": [
                 "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk/System/Library/Frameworks"
             ],
-            "compilerPath": "/opt/clang-9.0/bin/clang",
+            "compilerPath": "/opt/clang-10.0/bin/clang",
             "cStandard": "c11"
         }
     ],
     "version": 4
-}
\ No newline at end of file
+}

bazel/pl_build_system.bzl

@@ -28,7 +28,10 @@ def pl_copts():
     # Leaving this here as an example of how to add compiler dependent_flags.
     compiler_dependent_flags = select({
         "@pl//bazel:gcc_build": [],
-        "//conditions:default": [],
+        "//conditions:default": [
+            # TODO(yzhao/oazizi): Please remove this after fixing warnings in stirling.
+            "-Wno-c99-designator",
+        ],
     })
 
     return posix_options + manual_system_includes + tcmalloc_flags + compiler_dependent_flags

bazel/repositories.bzl

@@ -40,13 +40,13 @@ def _com_llvm_lib():
     native.new_local_repository(
         name = "com_llvm_lib",
         build_file = "third_party/llvm.BUILD",
-        path = "/opt/clang-9.0",
+        path = "/opt/clang-10.0",
     )
 
     native.new_local_repository(
         name = "com_llvm_lib_libcpp",
         build_file = "third_party/llvm.BUILD",
-        path = "/opt/clang-9.0-libc++",
+        path = "/opt/clang-10.0-libc++",
     )
 
 def _com_github_nghttp2_nghttp2():

chef/base_image_version.txt

-202007062118
+202007152344

chef/cookbooks/pixielabs/attributes/default.rb

 default['clang']               = {}
 default['clang']['deb']        =
-  'https://storage.googleapis.com/pl-infra-dev-artifacts/clang-9.0-pl5.deb'
+  'https://storage.googleapis.com/pl-infra-dev-artifacts/clang-10.0-pl1.deb'
 default['clang']['deb_sha256'] =
-  '6f261139154d4ffff9b3b5b2f049ab5e729df8d4fee1f55ffdb716a5d23a7e22'
-default['clang']['version'] = "9.0-pl3"
+  'abd6e0e32379fb83b359ddeb38ecd35fffb8dbf17e8450312c66bc86037d01a8'
+default['clang']['version'] = "10.0-pl1"
 
 default['gperftools']               = {}
 default['gperftools']['deb']        =

chef/cookbooks/pixielabs/metadata.rb

@@ -5,3 +5,4 @@ description      'Setup for pixielabs machines'
 version          '0.0.2'
 
 depends 'ark'
+depends 'alternatives'

chef/cookbooks/pixielabs/recipes/linux.rb

@@ -32,6 +32,10 @@ apt_pkg_list = [
   # Libtool/unwind, needed for perftools.
   'libltdl-dev',
   'libunwind-dev',
+
+  # Needed by Clang-10.
+  'libz3-4',
+  'libz3-dev',
 ]
 
 apt_package apt_pkg_list do
@@ -55,5 +59,12 @@ apt_package ['gcc-10','g++-10'] do
   action :upgrade
 end
 
+alternatives 'python install' do
+  link_name 'python'
+  path '/usr/bin/python3'
+  priority 100
+  action :install
+end
+
 include_recipe 'pixielabs::linux_java'
 include_recipe 'pixielabs::linux_clang'

chef/cookbooks/pixielabs/recipes/linux_clang.rb

@@ -14,7 +14,7 @@ file '/tmp/clang.deb' do
   action :delete
 end
 
-ENV['PATH'] = "/opt/clang-7.0/bin:#{ENV['PATH']}"
-ENV['LD_LIBRARY_PATH'] = "/opt/clang-7.0/lib:#{ENV['LD_LIBRARY_PATH']}"
+ENV['PATH'] = "/opt/clang-10.0/bin:#{ENV['PATH']}"
+ENV['LD_LIBRARY_PATH'] = "/opt/clang-10.0/lib:#{ENV['LD_LIBRARY_PATH']}"
 ENV['CC'] = "clang"
 ENV['CXX'] = "clang++"

chef/cookbooks/pixielabs/templates/plenv.inc.erb

 # If changing this file, make sure to also update the ENV in chef/packer_docker_dev.json.
 # DO NOT EDIT! This is an automatically generated file by Chef and may be changed.
-export CLANG_PATH=/opt/clang-9.0
+export CLANG_PATH=/opt/clang-10.0
 export PATH=/opt/google-cloud-sdk/bin:$PATH
 export PATH=/opt/node/bin:$PATH
 export PATH=/opt/phab/arcanist/bin:$PATH
@@ -13,7 +13,7 @@ export LD_LIBRARY_PATH=${CLANG_PATH}/lib:${LD_LIBRARY_PATH}
 export CC=clang
 export CXX=clang++
 
-export ASAN_SYMBOLIZER_PATH=/opt/clang-9.0/bin/llvm-symbolizer
+export ASAN_SYMBOLIZER_PATH=/opt/clang-10.0/bin/llvm-symbolizer
 
 if [ -n "`$SHELL -c 'echo $ZSH_VERSION'`" ]; then
    # Assume Zsh.

chef/packer_docker_dev.json

@@ -10,12 +10,12 @@
       "image": "{{ user `base_image_and_tag` }}",
       "commit": true,
       "changes": [
-        "ENV PATH /opt/golang/bin/:/opt/pixielabs/bin:/opt/clang-9.0/bin:/pl/bin:/opt/google-cloud-sdk/bin:/opt/node/bin:/opt/phab/arcanist/bin:/usr/lib/go-1.10/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-        "ENV LD_LIBRARY_PATH /opt/clang-9.0/lib",
+        "ENV PATH /opt/golang/bin/:/opt/pixielabs/bin:/opt/clang-10.0/bin:/pl/bin:/opt/google-cloud-sdk/bin:/opt/node/bin:/opt/phab/arcanist/bin:/usr/lib/go-1.10/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+        "ENV LD_LIBRARY_PATH /opt/clang-10.0/lib",
         "ENV GOPATH /pl",
         "ENV CC clang",
         "ENV CXX clang++",
-        "ENV ASAN_SYMBOLIZER_PATH /opt/clang-9.0/bin/llvm-symbolizer",
+        "ENV ASAN_SYMBOLIZER_PATH /opt/clang-10.0/bin/llvm-symbolizer",
         "WORKDIR /pl/src/pixielabs.ai/pixielabs"
       ]
     }

ci/jenkins.bazelrc

@@ -28,5 +28,5 @@ test --test_output=errors
 # Other test args.
 test --verbose_failures
 
-build:clang --action_env=CC=/opt/clang-9.0/bin/clang
-build:clang --action_env=CXX=/opt/clang-9.0/bin/clang++
+build:clang --action_env=CC=/opt/clang-10.0/bin/clang
+build:clang --action_env=CXX=/opt/clang-10.0/bin/clang++

docker.properties

-DOCKER_IMAGE_TAG=202006162125
+DOCKER_IMAGE_TAG=202007152354

docker/Makefile

@@ -7,7 +7,7 @@ SSH_PRIVATE_KEY_FILE := $(HOME)/.ssh/id_rsa
 SSH_PRIVATE_KEY := `cat $(SSH_PRIVATE_KEY_FILE)`
 
 ## Clang deb parameters
-CLANG_DEB_IMAGE_VERSION := 9.0-pl5
+CLANG_DEB_IMAGE_VERSION := 10.0-pl1
 clang_deb_fname := "clang-$(CLANG_DEB_IMAGE_VERSION).deb"
 clang_deb_gs_path :=  gs://pl-infra-dev-artifacts/$(clang_deb_fname)
 clang_deb_image_tag := "gcr.io/pl-dev-infra/clang_deb_image:$(CLANG_DEB_IMAGE_VERSION)"

docker/clang_deb_image/Dockerfile

-FROM cdrx/fpm-ubuntu:18.04
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y --fix-missing
+RUN apt-get install -y ruby ruby-dev rubygems build-essential
+RUN gem install --no-document fpm
 
-RUN apt-get update --fix-missing
 RUN apt-get install -y bison build-essential cmake flex git libedit-dev \
   clang libclang-dev llvm llvm-dev\
-  python2.7-dev swig libncurses5-dev zlib1g-dev libelf-dev subversion
+  python3 python3-distutils swig libncurses5-dev zlib1g-dev libelf-dev subversion
 
 ENV CC=clang
 ENV CXX=clang++
 
 WORKDIR /llvm_all
-RUN git clone --branch llvmorg-9.0.0 --depth 1 https://github.com/llvm/llvm-project.git
+RUN git clone --branch llvmorg-10.0.0 --depth 1 https://github.com/llvm/llvm-project.git
 
 WORKDIR /llvm_all/build
 RUN triple=$(gcc -v 2>&1 | grep "^Target:" | cut -d ' ' -f 2) && \
     cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release \
-    -DLLVM_BUILD_DOCS=OFF -DCMAKE_INSTALL_PREFIX=/opt/clang-9.0 \
+    -DLLVM_BUILD_DOCS=OFF -DCMAKE_INSTALL_PREFIX=/opt/clang-10.0 \
     -DLLVM_TARGETS_TO_BUILD="BPF;X86;WebAssembly" \
     -DLLVM_ENABLE_SPHINX=Off \
     -DLLVM_ENABLE_DOXYGEN=OFF \
@@ -30,7 +34,7 @@ RUN make -j $(nproc)
 RUN make install
 
 # We need libclang.a, but the clang build system excludes it during make install.
-RUN cp -a /llvm_all/build/lib/libclang.a /opt/clang-9.0/lib
+RUN cp -a /llvm_all/build/lib/libclang.a /opt/clang-10.0/lib
 
 #######################################################
 # This installs Clang/LLVM with libc++.
@@ -40,23 +44,22 @@ RUN cp -a /llvm_all/build/lib/libclang.a /opt/clang-9.0/lib
 # library.
 #######################################################
 WORKDIR /llvm_libc++
-RUN git clone https://github.com/rsmmr/install-clang
+RUN git clone https://github.com/pixie-labs/install-clang
 
 WORKDIR /llvm_libc++/install-clang
-RUN git checkout 4c25bb7486a786efd082ab3cb3abdcdb5c356bc4
-COPY install_clang.patch install_clang.patch
-RUN patch -p1 < install_clang.patch
-RUN ./install-clang -j $(nproc) -A /opt/clang-9.0-libc++ -C
-RUN rm -rf /opt/clang-9.0-libc++/src
+RUN git fetch
+RUN git checkout 978976c108249f542c189b7afa14a5a8c0204a3a
+RUN ./install-clang -j $(nproc) -A /opt/clang-10.0-libc++ -C
+RUN rm -rf /opt/clang-10.0-libc++/src
 
 # TODO(zasgar): MSAN version. It's currently failing to build because
 # of an MSAN failure inside of LLVM.
 #COPY msan.patch msan.patch
 #RUN patch -p1 < msan.patch
-#RUN ./install-clang -j $(nproc) -A /opt/clang-9.0-libc++-msan -C
+#RUN ./install-clang -j $(nproc) -A /opt/clang-10.0-libc++-msan -C
 
 WORKDIR /opt
 VOLUME /image
-ENV deb_name clang-9.0-pl5.deb
+ENV deb_name clang-10.0-pl1.deb
 CMD ["sh", "-c",  "fpm -p /image/${deb_name} \
-        -s dir -t deb -n clang-9.0 -v 9.0-pl5 --prefix /opt clang-9.0 clang-9.0-libc++"]
+        -s dir -t deb -n clang-10.0 -v 10.0-pl1 --prefix /opt clang-10.0 clang-10.0-libc++"]

docker/clang_deb_image/install_clang.patch

-diff --git a/install-clang b/install-clang
-index cb6f0fa..f1c512b 100755
---- a/install-clang
-+++ b/install-clang
-@@ -68,7 +68,7 @@ assertions=off             # If "on", enable LLVM assertions.
- parallelism=4              # The value X to pass to make -j X to build in parallel.
- buildtype=Release          # LLVM/clang build type.
- mode=install               # Install from scratch.
--targets=host               # LLVM_TARGETS_TO_BUILD ("all" builds them all).
-+targets="BPF;X86;WebAssembly"          # LLVM_TARGETS_TO_BUILD ("all" builds them all).
-
- # Whether to build & link shared libraries. Default is static, except for macOS
- # where static fails.
-@@ -320,12 +320,12 @@ projects_stage2="clang;libcxx;compiler-rt;libcxxabi;libunwind"
-
- test ${perform_lldb_build} = 1 && projects_stage2="${projects_stage2};lldb"
- test ${perform_lld_build} = 1 && projects_stage2="${projects_stage2};lld"
--test ${perform_extra_build} = 1 && projects_stage2="${projects_stage2};clang-tools-extra"
-+test ${perform_extra_build} = 1 && projects_stage2="${projects_stage2};clang-tools-extra;clangd"
-
--CMAKE_common="${CMAKE_common} -DBUILD_SHARED_LIBS=${build_shared_libs} -DLLVM_TARGETS_TO_BUILD=${targets}"
-+CMAKE_common="${CMAKE_common} -DBUILD_SHARED_LIBS=${build_shared_libs} -DLLVM_TARGETS_TO_BUILD=${targets} -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_TESTS=OFF -DLLVM_ENABLE_RTTI=ON"
- CMAKE_stage0="${CMAKE_common} ${CMAKE_stage0} -DLLVM_ENABLE_PROJECTS=${projects_stage0}"
- CMAKE_stage1="${CMAKE_common} ${CMAKE_stage1} -DLLVM_ENABLE_PROJECTS=${projects_stage1} -DLIBCXXABI_USE_LLVM_UNWINDER=YES -DLLVM_ENABLE_ASSERTIONS=${assertions}"
--CMAKE_stage2="${CMAKE_common} ${CMAKE_stage2} -DLLVM_ENABLE_PROJECTS=${projects_stage2} -DLIBCXXABI_USE_LLVM_UNWINDER=YES -DLLVM_ENABLE_ASSERTIONS=${assertions}"
-+CMAKE_stage2="${CMAKE_common} ${CMAKE_stage2} -DLLVM_ENABLE_PROJECTS=${projects_stage2} -DLIBCXXABI_USE_LLVM_UNWINDER=YES -DLIBCLANG_BUILD_STATIC=ON"
-
- #### Configure the stages.
-
-@@ -409,6 +409,8 @@ for stage in 0 1 2; do
-     fi
- done
-
-+cp "${src}/build-stage2/lib/libclang.a" "${prefix}/lib"
-+
- if [ "${perform_cleanup}" == "1" ]; then
-     echo Deleting $src ...
-     rm -rf "${src}"

scripts/run_clang_tidy.sh

@@ -55,11 +55,11 @@ fi
 # We can have the Clang tidy script in a few different places. Check them in priority
 # order.
 clang_tidy_full_scripts=(
-    "/opt/clang-9.0/share/clang/run-clang-tidy.py"
+    "/opt/clang-10.0/share/clang/run-clang-tidy.py"
     "/usr/local/opt/llvm/share/clang/run-clang-tidy.py")
 
 clang_tidy_diff_scripts=(
-    "/opt/clang-9.0/share/clang/clang-tidy-diff.py"
+    "/opt/clang-10.0/share/clang/clang-tidy-diff.py"
     "/usr/local/opt/llvm/share/clang/clang-tidy-diff.py")
 
 search_scripts="${clang_tidy_full_scripts[@]}"

src/carnot/exec/expression_evaluator.cc

@@ -127,7 +127,7 @@ Status ScalarExpressionEvaluator::Evaluate(ExecState* exec_state, const RowBatch
   CHECK(output != nullptr);
   CHECK_EQ(static_cast<size_t>(output->num_columns()), expressions_.size());
 
-  for (const auto expression : expressions_) {
+  for (const auto& expression : expressions_) {
     PL_RETURN_IF_ERROR(EvaluateSingleExpression(exec_state, input, *expression, output));
   }
   return Status::OK();
@@ -183,7 +183,7 @@ VectorNativeScalarExpressionEvaluator::EvaluateSingleExpression(
           const std::vector<types::SharedColumnWrapper>& children) -> types::SharedColumnWrapper {
         std::vector<types::DataType> arg_types;
         arg_types.reserve(children.size());
-        for (const auto child : children) {
+        for (const auto& child : children) {
           arg_types.emplace_back(child->data_type());
         }
 
@@ -192,7 +192,7 @@ VectorNativeScalarExpressionEvaluator::EvaluateSingleExpression(
 
         std::vector<const types::ColumnWrapper*> raw_children;
         raw_children.reserve(children.size());
-        for (const auto child : children) {
+        for (const auto& child : children) {
           raw_children.emplace_back(child.get());
         }
         auto output = types::ColumnWrapper::Make(def->exec_return_type(), num_rows);
@@ -274,7 +274,7 @@ Status exec::ArrowNativeScalarExpressionEvaluator::EvaluateSingleExpression(
           -> std::shared_ptr<arrow::Array> {
         std::vector<types::DataType> arg_types;
         arg_types.reserve(children.size());
-        for (const auto child : children) {
+        for (const auto& child : children) {
           arg_types.emplace_back(ArrowToDataType(child->type_id()));
         }
 

src/carnot/plan/scalar_expression.cc

@@ -201,7 +201,7 @@ Status ScalarFunc::Init(const planpb::ScalarFunc& pb) {
   DCHECK_EQ(pb.args_size(), pb.args_data_types_size());
   name_ = pb.name();
   udf_id_ = pb.id();
-  for (const auto arg : pb.args()) {
+  for (const auto& arg : pb.args()) {
     auto s = ScalarExpression::FromProto(arg);
     if (!s.ok()) {
       return s.status();
@@ -280,7 +280,7 @@ std::string ScalarFunc::DebugString() const {
 Status AggregateExpression::Init(const planpb::AggregateExpression& pb) {
   name_ = pb.name();
   uda_id_ = pb.id();
-  for (const auto arg : pb.args()) {
+  for (const auto& arg : pb.args()) {
     // arg is of message type AggregateExpression.Arg. Needs to be casted to a ScalarExpression.
     planpb::ScalarExpression se;