docker-compose: limit the amount of data copied

The AWS dependency includes a VERY large set of directories, which
are over 9G in size.

To avoid some of this data movement, we suggest shallow clones and
update our vendor rsync to exclude directories over 500M. This
drastically speeds up the copy and overall build time.

More investigation needs to be done, and perhaps a switch away from
git clones for this recipe as the data over the network during fetch
is still an issue.

Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>

recipes-containers/docker-compose/docker-compose_git.bb

@@ -78,3 +78,7 @@ FILES:${PN} += " ${nonarch_libdir}/docker/cli-plugins/"
 
 INHIBIT_PACKAGE_STRIP = "1"
 INSANE_SKIP:${PN} += "ldflags already-stripped"
+
+# the AWS dependency is 8GB, try and control the
+# size of the clones
+BB_GIT_SHALLOW = "1"

recipes-containers/docker-compose/relocation.inc

@@ -197,13 +197,39 @@ do_compile:prepend() {
         site_dest=$(echo $s | cut -d: -f1)
         site_source=$(echo $s | cut -d: -f2)
         force_flag=$(echo $s | cut -d: -f3)
+
         mkdir -p vendor.copy/$site_dest
+
+        # create a temporary exclude file
+        exclude_file=$(mktemp)
+
+        find vendor.fetch/$site_source -type d -print0 | \
+        xargs -0 du -sBM 2>/dev/null | \
+        awk '{if ($1+0 > 500) print substr($0, index($0,$2))}' | \
+        sed 's|^vendor.fetch/||' > "$exclude_file"
+
         if [ -n "$force_flag" ]; then
             echo "[INFO] $site_dest: force copying .go files"
             rm -rf vendor.copy/$site_dest
-            rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest
+            rsync -a \
+                --exclude='vendor/' \
+                --exclude='.git/' \
+                --exclude-from="$exclude_file" \
+                vendor.fetch/$site_source/ vendor.copy/$site_dest
         else
-            [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; }
+            if [ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ]; then
+                echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)"
+                true
+            else
+                echo "[INFO] $site_dest: copying .go files"
+                rsync -a \
+                    --exclude='vendor/' \
+                    --exclude='.git/' \
+                    --exclude-from="$exclude_file" \
+                    vendor.fetch/$site_source/ vendor.copy/$site_dest
+            fi
         fi
+
+        rm -f "$exclude_file"
     done
 }