recipetool/devtool: calculate source paths relative to UNPACKDIR

Now that recipes default to S in UNPACKDIR, recipetool and devtool should
do the same.

There was some discussion about changing devtool to simply setting
UNPACKDIR via bbappend to a workspace and running unpack task directly;
currently it has a bunch of convoluted path calculations, substitutions,
moving source trees around and and special casing (devtool-source.bbclass
in particular is an unpleasant hack).

This should definitely be done; but right now we can simply tweak existing
code which at least doesn't make it worse.

(From OE-Core rev: c326ca8aeb2bf0f7719e43921d10efd5dedc7b2a)

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes/devtool-source.bbclass

@@ -92,9 +92,9 @@ python devtool_post_unpack() {
             for fname in local_files:
                 f.write('%s\n' % fname)
 
-    if os.path.dirname(srcsubdir) != workdir:
+    if srcsubdir.startswith(unpackdir) and os.path.dirname(srcsubdir) != unpackdir:
         # Handle if S is set to a subdirectory of the source
-        srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
+        srcsubdir = os.path.normpath(os.path.join(unpackdir, os.path.relpath(srcsubdir, unpackdir).split(os.sep)[0]))
 
     scriptutils.git_convert_standalone_clone(srcsubdir)
 

meta/lib/oeqa/selftest/cases/devtool.py

@@ -565,7 +565,7 @@ class DevtoolAddTests(DevtoolBase):
         recipefile = get_bb_var('FILE', testrecipe)
         self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
+        checkvars['S'] = '${UNPACKDIR}/MarkupSafe-${PV}'
         checkvars['SRC_URI'] = url.replace(testver, '${PV}')
         self._test_recipe_contents(recipefile, checkvars, [])
         # Try with version specified
@@ -582,7 +582,7 @@ class DevtoolAddTests(DevtoolBase):
         recipefile = get_bb_var('FILE', testrecipe)
         self.assertIn('%s_%s.bb' % (testrecipe, fakever), recipefile, 'Recipe file incorrectly named')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
+        checkvars['S'] = '${UNPACKDIR}/MarkupSafe-%s' % testver
         checkvars['SRC_URI'] = url
         self._test_recipe_contents(recipefile, checkvars, [])
 
@@ -1627,12 +1627,12 @@ class DevtoolUpdateTests(DevtoolBase):
         # Check preconditions
         testrecipe = 'dos2unix'
         self.append_config('ERROR_QA:remove:pn-dos2unix = "patch-status"\n')
-        bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE', 'BB_GIT_DEFAULT_DESTSUFFIX'], testrecipe)
+        bb_vars = get_bb_vars(['SRC_URI', 'S', 'UNPACKDIR', 'FILE', 'BB_GIT_DEFAULT_DESTSUFFIX'], testrecipe)
         self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe)
-        workdir_git = '%s/%s/' % (bb_vars['WORKDIR'], bb_vars['BB_GIT_DEFAULT_DESTSUFFIX'])
+        unpackdir_git = '%s/%s/' % (bb_vars['UNPACKDIR'], bb_vars['BB_GIT_DEFAULT_DESTSUFFIX'])
-        if not bb_vars['S'].startswith(workdir_git):
+        if not bb_vars['S'].startswith(unpackdir_git):
             self.fail('This test expects the %s recipe to be building from a subdirectory of the git repo' % testrecipe)
-        subdir = bb_vars['S'].split(workdir_git, 1)[1]
+        subdir = bb_vars['S'].split(unpackdir_git, 1)[1]
         # Clean up anything in the workdir/sysroot/sstate cache
         bitbake('%s -c cleansstate' % testrecipe)
         # Try modifying a recipe
@@ -2414,7 +2414,7 @@ class DevtoolUpgradeTests(DevtoolBase):
         newsrctree = os.path.join(self.workspacedir, 'sources', newrecipename)
         self.assertExists(newsrctree, 'Source directory not renamed')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/%s-%s' % (recipename, recipever)
+        checkvars['S'] = '${UNPACKDIR}/%s-%s' % (recipename, recipever)
         checkvars['SRC_URI'] = url
         self._test_recipe_contents(newrecipefile, checkvars, [])
         # Try again - change just name this time
@@ -2426,7 +2426,7 @@ class DevtoolUpgradeTests(DevtoolBase):
         self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipename), 'Old recipe directory still exists')
         self.assertExists(os.path.join(self.workspacedir, 'sources', newrecipename), 'Source directory not renamed')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/%s-${PV}' % recipename
+        checkvars['S'] = '${UNPACKDIR}/%s-${PV}' % recipename
         checkvars['SRC_URI'] = url.replace(recipever, '${PV}')
         self._test_recipe_contents(newrecipefile, checkvars, [])
         # Try again - change just version this time
@@ -2437,7 +2437,7 @@ class DevtoolUpgradeTests(DevtoolBase):
         self.assertExists(newrecipefile, 'Recipe file not renamed')
         self.assertExists(os.path.join(self.workspacedir, 'sources', recipename), 'Source directory no longer exists')
         checkvars = {}
-        checkvars['S'] = '${WORKDIR}/${BPN}-%s' % recipever
+        checkvars['S'] = '${UNPACKDIR}/${BPN}-%s' % recipever
         checkvars['SRC_URI'] = url
         self._test_recipe_contents(newrecipefile, checkvars, [])
 

meta/lib/oeqa/selftest/cases/recipetool.py

@@ -1144,10 +1144,10 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase):
 
     def test_recipetool_appendsrcfile_srcdir_basic(self):
         testrecipe = 'bash'
-        bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe)
+        bb_vars = get_bb_vars(['S', 'UNPACKDIR'], testrecipe)
         srcdir = bb_vars['S']
-        workdir = bb_vars['WORKDIR']
+        unpackdir = bb_vars['UNPACKDIR']
-        subdir = os.path.relpath(srcdir, workdir)
+        subdir = os.path.relpath(srcdir, unpackdir)
         self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir)
 
     def test_recipetool_appendsrcfile_existing_in_src_uri(self):
@@ -1196,10 +1196,10 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase):
     def test_recipetool_appendsrcfile_replace_file_srcdir(self):
         testrecipe = 'bash'
         filepath = 'Makefile.in'
-        bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe)
+        bb_vars = get_bb_vars(['S', 'UNPACKDIR'], testrecipe)
         srcdir = bb_vars['S']
-        workdir = bb_vars['WORKDIR']
+        unpackdir = bb_vars['UNPACKDIR']
-        subdir = os.path.relpath(srcdir, workdir)
+        subdir = os.path.relpath(srcdir, unpackdir)
 
         self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir)
         bitbake('%s:do_unpack' % testrecipe)

scripts/lib/devtool/ide_sdk.py

@@ -334,7 +334,7 @@ class RecipeModified:
         self.srctree = workspace[workspacepn]['srctree']
         # Need to grab this here in case the source is within a subdirectory
         self.real_srctree = get_real_srctree(
-            self.srctree, recipe_d.getVar('S'), recipe_d.getVar('WORKDIR'))
+            self.srctree, recipe_d.getVar('S'), recipe_d.getVar('UNPACKDIR'))
         self.bbappend = workspace[workspacepn]['bbappend']
 
         self.ide_sdk_dir = os.path.join(

scripts/lib/devtool/standard.py

@@ -625,7 +625,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
                 srcsubdir = f.read()
         except FileNotFoundError as e:
             raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
-        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
+        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir', os.path.relpath(d.getVar('UNPACKDIR'), d.getVar('WORKDIR'))))
 
         # Check if work-shared is empty, if yes
         # find source and copy to work-shared
@@ -742,13 +742,13 @@ def get_staging_kbranch(srcdir):
         staging_kbranch = "".join(branch.split('\n')[0])
     return staging_kbranch
 
-def get_real_srctree(srctree, s, workdir):
+def get_real_srctree(srctree, s, unpackdir):
     # Check that recipe isn't using a shared workdir
     s = os.path.abspath(s)
-    workdir = os.path.abspath(workdir)
+    unpackdir = os.path.abspath(unpackdir)
-    if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
+    if s.startswith(unpackdir) and s != unpackdir and os.path.dirname(s) != unpackdir:
         # Handle if S is set to a subdirectory of the source
-        srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
+        srcsubdir = os.path.relpath(s, unpackdir).split(os.sep, 1)[1]
         srctree = os.path.join(srctree, srcsubdir)
     return srctree
 
@@ -907,7 +907,7 @@ def modify(args, config, basepath, workspace):
 
         # Need to grab this here in case the source is within a subdirectory
         srctreebase = srctree
-        srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
+        srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('UNPACKDIR'))
 
         bb.utils.mkdirhier(os.path.dirname(appendfile))
         with open(appendfile, 'w') as f:

scripts/lib/devtool/upgrade.py

@@ -571,7 +571,7 @@ def upgrade(args, config, basepath, workspace):
         else:
             srctree = standard.get_default_srctree(config, pn)
 
-        srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
+        srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('UNPACKDIR'))
 
         # try to automatically discover latest version and revision if not provided on command line
         if not args.version and not args.srcrev:

scripts/lib/recipetool/append.py

@@ -317,7 +317,7 @@ def appendsrc(args, files, rd, extralines=None):
     import oe.recipeutils
 
     srcdir = rd.getVar('S')
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
 
     import bb.fetch
     simplified = {}
@@ -336,10 +336,10 @@ def appendsrc(args, files, rd, extralines=None):
         src_destdir = os.path.dirname(srcfile)
         if not args.use_workdir:
             if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
-                srcdir = os.path.join(workdir, rd.getVar('BB_GIT_DEFAULT_DESTSUFFIX'))
+                srcdir = os.path.join(unpackdir, rd.getVar('BB_GIT_DEFAULT_DESTSUFFIX'))
                 if not bb.data.inherits_class('kernel-yocto', rd):
-                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/${BB_GIT_DEFAULT_DESTSUFFIX}')
+                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${UNPACKDIR}/${BB_GIT_DEFAULT_DESTSUFFIX}')
-            src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
+            src_destdir = os.path.join(os.path.relpath(srcdir, unpackdir), src_destdir)
         src_destdir = os.path.normpath(src_destdir)
 
         if src_destdir and src_destdir != '.':

scripts/lib/recipetool/create.py

@@ -735,7 +735,7 @@ def create_recipe(args):
     if srcsubdir and not args.binary:
         # (for binary packages we explicitly specify subdir= when fetching to
         # match the default value of S, so we don't need to set it in that case)
-        lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
+        lines_before.append('S = "${UNPACKDIR}/%s"' % srcsubdir)
         lines_before.append('')
 
     if pkgarch:
@@ -839,7 +839,7 @@ def create_recipe(args):
                 line = line.replace(realpv, '${PV}')
             if pn:
                 line = line.replace(pn, '${BPN}')
-            if line == 'S = "${WORKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
+            if line == 'S = "${UNPACKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
                 skipblank = True
                 continue
         elif line.startswith('SRC_URI = '):