Skip to content

Commit b715625

Browse files
prabhuprabhu
authored
Recurse on optional package tree (#1860)
* Recurse on optional package tree Signed-off-by: Prabhu Subramanian <[email protected]> * Update packages Signed-off-by: Prabhu Subramanian <[email protected]> * Ruby image fixes Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
1 parent de9c715 commit b715625

File tree

10 files changed

+157
-127
lines changed

10 files changed

+157
-127
lines changed

‎ci/images/debian/Dockerfile.ruby26‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ ENV PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:${PATH}:/usr/local/bin:/r
1717
COPY ci/images/debian/install.sh /tmp/
1818

1919
RUN apt-get update && apt-get install -qq -y --no-install-recommends curl bash bzip2 git-core zip unzip make gawk \
20-
&& apt-get install -qq -y build-essential gcc-9 g++-9 python2 libmagic-dev locales nodejs \
20+
&& apt-get install -qq -y build-essential gcc-10 g++-10 python3 python3-pip python3-dev libmagic-dev locales \
2121
&& gem install bundler -v 1.17.3 \
2222
&& bundle config git.allow_insecure true \
2323
&& chmod +x /tmp/install.sh \

‎ci/images/debian/Dockerfile.ruby33‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ENV PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:/usr/local/bin:/r
1616
COPY ci/images/debian/install.sh /tmp/
1717

1818
RUN apt-get update && apt-get install -qq -y --no-install-recommends curl bash bzip2 git-core zip unzip make gawk \
19-
&& apt-get install -qq -y build-essential python3 python3-pip python3-dev libmagic-dev locales \
19+
&& apt-get install -qq -y build-essential python3 python3-pip python3-dev libmagic-dev locales libffi-dev \
2020
&& chmod +x /tmp/install.sh \
2121
&& /tmp/install.sh && rm /tmp/install.sh \
2222
&& node -v \

‎ci/images/debian/Dockerfile.ruby34‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ ENV PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:/usr/local/bin:/r
1414
COPY ci/images/debian/install.sh /tmp/
1515

1616
RUN apt-get update && apt-get install -qq -y --no-install-recommends curl bash bzip2 git-core zip unzip make gawk \
17-
&& apt-get install -qq -y build-essential python3 python3-pip python3-dev libmagic-dev locales \
17+
&& apt-get install -qq -y build-essential python3 python3-pip python3-dev libmagic-dev locales libffi-dev \
1818
&& chmod +x /tmp/install.sh \
1919
&& /tmp/install.sh && rm /tmp/install.sh \
2020
&& node -v \

‎contrib/piptree.py‎

Lines changed: 48 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -42,48 +42,69 @@ def get_installed_distributions(python_path=None):
4242
return [d._dist for d in dists]
4343

4444

45+
def _get_extra_deps_from_dist(dist):
46+
extra_deps = {}
47+
if not dist:
48+
return extra_deps
49+
# all requirements, some of which may be extra-only:
50+
reqs = dist.metadata.get_all('Requires-Dist') or []
51+
# extras this package defines:
52+
extras = dist.metadata.get_all('Provides-Extra') or []
53+
for req_str in reqs:
54+
req = Requirement(req_str)
55+
if req.marker and 'extra' in str(req.marker):
56+
# evaluate marker for each declared extra
57+
for extra in extras:
58+
if req.marker.evaluate({'extra': extra}):
59+
extra_deps.setdefault(extra, []).append({"name": str(req.name), "versionSpecifiers": str(req.specifier), "url": str(req.url) if req.url else None})
60+
return extra_deps
61+
62+
63+
def _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps):
64+
dependencies = []
65+
if not extra_deps:
66+
return dependencies
67+
# Treat an extra with the name all as dependencies
68+
all_deps = extra_deps.get("all", [])
69+
for dep in all_deps:
70+
dversion = name_version_cache.get(dep["name"])
71+
if not dversion:
72+
continue
73+
dversionSpecifiers = dep.get("versionSpecifiers")
74+
dpurl = f"""pkg:pypi/{dep["name"].lower()}@{dversion}"""
75+
dextra_deps = _get_extra_deps_from_dist(name_dist_cache.get(dep["name"]))
76+
ddependencies = _get_deps_from_extras(name_version_cache, name_dist_cache, dextra_deps)
77+
dependencies.append({
78+
"name": dep["name"],
79+
"version": dversion,
80+
"versionSpecifiers": dversionSpecifiers,
81+
"purl": dpurl,
82+
"extra_deps": dextra_deps,
83+
"dependencies": ddependencies
84+
})
85+
return dependencies
86+
87+
4588
def get_installed_with_extras():
4689
result = {}
4790
if not REQUIREMENT_MODULE_FOUND:
4891
return result
4992
name_version_cache = {}
93+
name_dist_cache = {}
5094
for dist in importlib_metadata.distributions():
5195
name = dist.metadata['Name']
5296
version = dist.version or ""
5397
name_version_cache[name] = version
98+
name_dist_cache[name] = dist
5499
for dist in importlib_metadata.distributions():
55100
name = dist.metadata['Name']
56101
version = dist.version or ""
57102
# extras this package defines:
58103
extras = dist.metadata.get_all('Provides-Extra') or []
59-
# all requirements, some of which may be extra-only:
60-
reqs = dist.metadata.get_all('Requires-Dist') or []
61-
62104
# map each extra → its extra-only dependencies
63-
extra_deps = {}
64-
for req_str in reqs:
65-
req = Requirement(req_str)
66-
if req.marker and 'extra' in str(req.marker):
67-
# evaluate marker for each declared extra
68-
for extra in extras:
69-
if req.marker.evaluate({'extra': extra}):
70-
extra_deps.setdefault(extra, []).append({"name": str(req.name), "versionSpecifiers": str(req.specifier), "url": str(req.url) if req.url else None})
105+
extra_deps = _get_extra_deps_from_dist(dist)
71106
purl = f"pkg:pypi/{name.lower()}@{version}"
72-
dependencies = []
73-
# Treat an extra with the name all as dependencies
74-
all_deps = extra_deps.get("all", [])
75-
for dep in all_deps:
76-
dversion = name_version_cache.get(dep["name"])
77-
if not dversion:
78-
continue
79-
dversionSpecifiers = dep.get("versionSpecifiers")
80-
dpurl = f"""pkg:pypi/{dep["name"].lower()}@{dversion}"""
81-
dependencies.append({
82-
"name": dep["name"],
83-
"version": dversion,
84-
"versionSpecifiers": dversionSpecifiers,
85-
"purl": dpurl
86-
})
107+
dependencies = _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps)
87108
result[purl] = {
88109
'name': name,
89110
'version': version,
@@ -162,15 +183,8 @@ def main(argv):
162183
"dependencies": dependencies + all_dependencies,
163184
}
164185
)
165-
all_deps = {}
166-
for t in tree:
167-
for d in t["dependencies"]:
168-
all_deps[d["name"]] = True
169-
trimmed_tree = [
170-
t for t in tree if t["name"] not in all_deps
171-
]
172186
with open(out_file, mode="w", encoding="utf-8") as fp:
173-
json.dump(trimmed_tree, fp)
187+
json.dump(tree, fp)
174188

175189

176190
if __name__ == "__main__":

‎deno.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,12 @@
6464
"@appthreat/cdx-proto": "npm:@appthreat/[email protected]",
6565
"@babel/parser": "npm:@babel/parser@^7.27.2",
6666
"@babel/traverse": "npm:@babel/traverse@^7.27.1",
67-
"@npmcli/arborist": "npm:@npmcli/arborist@^9.1.0",
67+
"@npmcli/arborist": "npm:@npmcli/arborist@^9.1.2",
6868
"ajv": "npm:ajv@^8.16.0",
6969
"ajv-formats": "npm:ajv-formats@^3.0.1",
7070
"cheerio": "npm:cheerio@^1.1.0",
7171
"edn-data": "npm:[email protected]",
72-
"glob": "npm:glob@^11.0.2",
72+
"glob": "npm:glob@^11.0.3",
7373
"global-agent": "npm:global-agent@^3.0.0",
7474
"got": "npm:got@^14.4.5",
7575
"iconv-lite": "npm:iconv-lite@^0.6.3",

‎lib/managers/piptree.js‎

Lines changed: 48 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -59,48 +59,69 @@ def get_installed_distributions(python_path=None):
5959
return [d._dist for d in dists]
6060
6161
62+
def _get_extra_deps_from_dist(dist):
63+
extra_deps = {}
64+
if not dist:
65+
return extra_deps
66+
# all requirements, some of which may be extra-only:
67+
reqs = dist.metadata.get_all('Requires-Dist') or []
68+
# extras this package defines:
69+
extras = dist.metadata.get_all('Provides-Extra') or []
70+
for req_str in reqs:
71+
req = Requirement(req_str)
72+
if req.marker and 'extra' in str(req.marker):
73+
# evaluate marker for each declared extra
74+
for extra in extras:
75+
if req.marker.evaluate({'extra': extra}):
76+
extra_deps.setdefault(extra, []).append({"name": str(req.name), "versionSpecifiers": str(req.specifier), "url": str(req.url) if req.url else None})
77+
return extra_deps
78+
79+
80+
def _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps):
81+
dependencies = []
82+
if not extra_deps:
83+
return dependencies
84+
# Treat an extra with the name all as dependencies
85+
all_deps = extra_deps.get("all", [])
86+
for dep in all_deps:
87+
dversion = name_version_cache.get(dep["name"])
88+
if not dversion:
89+
continue
90+
dversionSpecifiers = dep.get("versionSpecifiers")
91+
dpurl = f"""pkg:pypi/{dep["name"].lower()}@{dversion}"""
92+
dextra_deps = _get_extra_deps_from_dist(name_dist_cache.get(dep["name"]))
93+
ddependencies = _get_deps_from_extras(name_version_cache, name_dist_cache, dextra_deps)
94+
dependencies.append({
95+
"name": dep["name"],
96+
"version": dversion,
97+
"versionSpecifiers": dversionSpecifiers,
98+
"purl": dpurl,
99+
"extra_deps": dextra_deps,
100+
"dependencies": ddependencies
101+
})
102+
return dependencies
103+
104+
62105
def get_installed_with_extras():
63106
result = {}
64107
if not REQUIREMENT_MODULE_FOUND:
65108
return result
66109
name_version_cache = {}
110+
name_dist_cache = {}
67111
for dist in importlib_metadata.distributions():
68112
name = dist.metadata['Name']
69113
version = dist.version or ""
70114
name_version_cache[name] = version
115+
name_dist_cache[name] = dist
71116
for dist in importlib_metadata.distributions():
72117
name = dist.metadata['Name']
73118
version = dist.version or ""
74119
# extras this package defines:
75120
extras = dist.metadata.get_all('Provides-Extra') or []
76-
# all requirements, some of which may be extra-only:
77-
reqs = dist.metadata.get_all('Requires-Dist') or []
78-
79121
# map each extra → its extra-only dependencies
80-
extra_deps = {}
81-
for req_str in reqs:
82-
req = Requirement(req_str)
83-
if req.marker and 'extra' in str(req.marker):
84-
# evaluate marker for each declared extra
85-
for extra in extras:
86-
if req.marker.evaluate({'extra': extra}):
87-
extra_deps.setdefault(extra, []).append({"name": str(req.name), "versionSpecifiers": str(req.specifier), "url": str(req.url) if req.url else None})
122+
extra_deps = _get_extra_deps_from_dist(dist)
88123
purl = f"pkg:pypi/{name.lower()}@{version}"
89-
dependencies = []
90-
# Treat an extra with the name all as dependencies
91-
all_deps = extra_deps.get("all", [])
92-
for dep in all_deps:
93-
dversion = name_version_cache.get(dep["name"])
94-
if not dversion:
95-
continue
96-
dversionSpecifiers = dep.get("versionSpecifiers")
97-
dpurl = f"""pkg:pypi/{dep["name"].lower()}@{dversion}"""
98-
dependencies.append({
99-
"name": dep["name"],
100-
"version": dversion,
101-
"versionSpecifiers": dversionSpecifiers,
102-
"purl": dpurl
103-
})
124+
dependencies = _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps)
104125
result[purl] = {
105126
'name': name,
106127
'version': version,
@@ -179,15 +200,8 @@ def main(argv):
179200
"dependencies": dependencies + all_dependencies,
180201
}
181202
)
182-
all_deps = {}
183-
for t in tree:
184-
for d in t["dependencies"]:
185-
all_deps[d["name"]] = True
186-
trimmed_tree = [
187-
t for t in tree if t["name"] not in all_deps
188-
]
189203
with open(out_file, mode="w", encoding="utf-8") as fp:
190-
json.dump(trimmed_tree, fp)
204+
json.dump(tree, fp)
191205
192206
193207
if __name__ == "__main__":

‎package.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,12 +83,12 @@
8383
"@babel/parser": "^7.27.4",
8484
"@babel/traverse": "^7.27.4",
8585
"@iarna/toml": "2.2.5",
86-
"@npmcli/arborist": "^9.1.1",
86+
"@npmcli/arborist": "^9.1.2",
8787
"ajv": "^8.17.1",
8888
"ajv-formats": "^3.0.1",
8989
"cheerio": "^1.1.0",
9090
"edn-data": "1.1.2",
91-
"glob": "^11.0.2",
91+
"glob": "^11.0.3",
9292
"global-agent": "^3.0.0",
9393
"got": "^14.4.7",
9494
"iconv-lite": "^0.6.3",
@@ -160,7 +160,7 @@
160160
"istanbul-lib-instrument": "^6.0.3",
161161
"json-parse-even-better-errors": "^4.0.0",
162162
"lru-cache": "^11.1.0",
163-
"minimatch": "^10.0.1",
163+
"minimatch": "^10.0.3",
164164
"minizlib": "^3.0.2",
165165
"mkdirp": "^3.0.1",
166166
"ms": "^2.1.3",
@@ -209,7 +209,7 @@
209209
"istanbul-lib-instrument": "^6.0.3",
210210
"json-parse-even-better-errors": "^4.0.0",
211211
"lru-cache": "^11.1.0",
212-
"minimatch": "^10.0.1",
212+
"minimatch": "^10.0.3",
213213
"minizlib": "^3.0.2",
214214
"mkdirp": "^3.0.1",
215215
"ms": "^2.1.3",

0 commit comments

Comments
 (0)