Commit 02dd59fd authored by lynxis lazus's avatar lynxis lazus Committed by Holger Levsen

add reproducible_openwrt_package_parser.py

Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent 74fc1f1d
......@@ -644,6 +644,25 @@ else
MAGIC_SIGN="?"
fi
# parse the OpenWrt Packages.manifest files to generate results to fill into the database
for i in * ; do
cd "$i"
# search packages in both paths to find non-existing ones
PKGS1=$(find -- * -type f -name "Packages.manifest" | sort -u )
pushd "$RESULTSDIR/b2/$i"
PKGS2=$(find -- * -type f -name "Packages.manifest" | sort -u )
popd
for j in $(printf "%s\n%s" "$PKGS1" "$PKGS2" | sort -u ) ; do
if [ ! -f "$RESULTSDIR/b1/$i/$j" ] || [ ! -f "$RESULTSDIR/b2/$i/$j" ] ; then
echo "One Packages.manifest list does not exist! Either $RESULTSDIR/b1/$i/$j or $RESULTSDIR/b2/$i/$j"
fi
echo "Running reproducible_openwrt_package_parser.py"
/srv/jenkins/bin/reproducible_openwrt_package_parser.py "$PKGS1" "$PKGS2" || true
done
done
write_openwrt_page_header(){
cat > "$PAGE" <<- EOF
<!DOCTYPE html>
......
#!/usr/bin/env python
# Copyright Alexander Couzens <lynxis@fe80.eu> 2018
#
# under the GPL2
#
# Parse two Packages.manifest or Packages list from two builds and
# fill the database of reproducible-builds with the results
import email.parser
def download_manifest():
import requests
url = "https://downloads.openwrt.org/releases/18.06.1/targets/ramips/mt76x8/packages/Packages.manifest"
response = requests.get(url)
return str(response.content, 'utf-8')
def parse_packages(package_list_fp):
linebuffer = ""
packages = []
for line in package_list_fp:
if line == '\n':
parser = email.parser.Parser()
package = parser.parsestr(linebuffer)
packages.append(package)
linebuffer = ""
else:
linebuffer += line
return packages
def show_list_difference(list_a, list_b):
""" get two list of manifest and generate a result """
pkg_a = parse_packages(list_a)
pkg_b = parse_packages(list_b)
# packages which does not have the same pkg in B
alone_a = {}
# packages which does not have the same pkg in A
alone_b = {}
# package which are not reproducible
differ = {}
same = {}
dict_a = {}
dict_b = {}
for pkg in pkg_a:
dict_a[pkg['Package']] = pkg
for pkg in pkg_b:
dict_b[pkg['Package']] = pkg
for name in dict_a:
if name not in dict_b:
alone_a[name] = dict_a[name]
else:
if dict_a[name]['SHA256sum'] != dict_b[name]['SHA256sum']:
differ[name] = [dict_a[name], dict_b[name]]
else:
same[name] = [dict_a[name], dict_b[name]]
del dict_b[name]
for name in dict_b:
alone_b[name] = dict_b[name]
return (same, alone_a, alone_b, differ)
def insert_into_db(result):
from pprint import pprint
same, alone_a, alone_b, differ = result
for pkg in same:
pprint(pkg)
def example():
import io
package_list = io.StringIO(download_manifest)
packages = parse_packages(package_list)
for pkg in packages:
print(pkg['Filename'])
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('packagea', nargs=1, type=argparse.FileType('r'),
help='The first package list')
parser.add_argument('packageb', nargs=1, type=argparse.FileType('r'),
help='The second package list')
args = parser.parse_args()
result = show_list_difference(args.packagea[0], args.packageb[0])
insert_into_db(result)
if __name__ == "__main__":
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment