From b27566a3808ee2d11206a3f0a63f0b473f289372 Mon Sep 17 00:00:00 2001
From: vi <vi@computational.law>
Date: Fri, 8 Dec 2017 17:43:55 +0100
Subject: [PATCH] Naive package manager statistics.

---
 Makefile           | 24 ++++++++++++++++++++++++
 README.org         |  3 +++
 Statistics.hs      | 32 ++++++++++++++++++++++++++++++++
 expand-mirrors.py  | 35 +++++++++++++++++++++++++++++++++++
 extract-sources.sh | 11 +++++++++++
 stack.yaml         | 11 +++++++++++
 6 files changed, 116 insertions(+)
 create mode 100644 Makefile
 create mode 100644 README.org
 create mode 100644 Statistics.hs
 create mode 100644 expand-mirrors.py
 create mode 100644 extract-sources.sh
 create mode 100644 stack.yaml

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..8b5207a
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,24 @@
+%.result: %.urls Statistics
+	grep -E '^http://' $< | ./Statistics > $@
+
+Statistics: Statistics.hs stack.yaml
+	stack build https-everywhere-rules
+	stack build foldl
+	stack exec -- ghc $< -o $@
+
+arch-extra.urls: arch-extra extract-sources.sh
+	find $< -name PKGBUILD | ./extract-sources.sh | grep -E '^https?://' | tee $@
+
+mirrors.json: nixpkgs
+	nix-instantiate --eval --json --strict -E 'import ./$</pkgs/build-support/fetchurl/mirrors.nix' | tee $@
+
+expand-mirrors.py: mirrors.json
+
+nixpkgs.urls: nixpkgs expand-mirrors.py
+	nix-instantiate --eval --json --strict -E '(let pkgs = import ./$< { config.allowBroken = true; config.allowUnfree = true; }; in with pkgs.lib; flatten (mapAttrsToList (_: v: (builtins.tryEval v.src.url or []).value) pkgs))' | jq -r .[] | ./expand-mirrors.py | grep -E '^https?://' | tee $@
+
+arch-extra:
+	git clone https://git.archlinux.org/svntogit/packages.git $@
+
+nixpkgs:
+	git clone https://github.com/nixos/nixpkgs
diff --git a/README.org b/README.org
new file mode 100644
index 0000000..d62dbc1
--- /dev/null
+++ b/README.org
@@ -0,0 +1,3 @@
+Of the 11919 HTTP source references in the Nix Packages collection, 2098 (18%) could be upgraded automatically.
+
+Of the 2154 HTTP source references in Arch Linux's primary repository ('extra'), 491 (23%) could be upgraded automatically.
diff --git a/Statistics.hs b/Statistics.hs
new file mode 100644
index 0000000..e7ed7ac
--- /dev/null
+++ b/Statistics.hs
@@ -0,0 +1,32 @@
+module Main (main) where
+
+import Network.URI (URI, parseURI)
+import Data.HTTPSEverywhere.Rules (rewriteURL, RuleSet, getRulesets)
+import Pipes (Pipe, Producer, lift, yield, await, (>->))
+import Pipes.Prelude (stdinLn)
+import qualified Pipes.Prelude as Pipes (fold)
+import Control.Monad (forever)
+import Control.Foldl (Fold(..))
+import qualified Control.Foldl as Foldl
+import Data.Monoid (Sum(..))
+import Data.Bool (bool)
+
+parse :: Monad m => Pipe String URI m ()
+parse = forever $ parseURI <$> await >>= maybe (return ()) yield
+
+check :: [RuleSet] -> Pipe URI Bool IO ()
+check rules = forever $ do
+  src <- await
+  tgt <- lift $ rewriteURL rules src
+  yield $ src /= tgt
+
+fold :: Monad m => Fold a b -> Producer a m () -> m b
+fold (Fold step begin done) = Pipes.fold step begin done
+
+proportion :: Fold Bool (Int, Int)
+proportion = (,) <$> Foldl.foldMap (Sum . bool 0 1) getSum <*> Foldl.length
+
+main :: IO ()
+main = do
+  rules <- getRulesets
+  fold proportion (stdinLn >-> parse >-> check rules) >>= print
diff --git a/expand-mirrors.py b/expand-mirrors.py
new file mode 100644
index 0000000..9834049
--- /dev/null
+++ b/expand-mirrors.py
@@ -0,0 +1,35 @@
+#! /usr/bin/env python
+
+import sys, json, subprocess, logging, urlparse
+
+class Translator:
+    def __init__(self):
+        with open('mirrors.json') as mirrors:
+            self.mirrors = json.load(mirrors)
+    def interpret(self, url):
+        parsed_url = urlparse.urlparse(url)
+        if parsed_url.scheme == 'mirror':
+            if parsed_url.netloc in self.mirrors:
+                pidgin = []
+                for mirror in self.mirrors[parsed_url.netloc]:
+                    parsed_mirror = urlparse.urlparse(mirror)
+                    expansion = urlparse.ParseResult(
+                        parsed_mirror.scheme,
+                        parsed_mirror.netloc,
+                        parsed_url.path,
+                        parsed_url.params,
+                        parsed_url.query,
+                        parsed_url.fragment)
+                    pidgin.append(expansion.geturl())
+                return pidgin
+            else:
+                logging.warning("Unhandled mirror {} ({})".format(parsed_url.netloc, url))
+                return []
+        else:
+            return [ url ]
+        
+if __name__ == '__main__':
+    translator = Translator()
+    for line in sys.stdin:
+        for translation in translator.interpret(line.strip()):
+            print translation
diff --git a/extract-sources.sh b/extract-sources.sh
new file mode 100644
index 0000000..8143de4
--- /dev/null
+++ b/extract-sources.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+while read PKGBUILD; do
+		TMP="$(mktemp)";
+		awk -F= '
+				BEGIN {state=0}
+				(NR > 1 && state == 0){print;if($1 == "source"){state = 1};next;}
+				(state == 1){ if(match($1,"^[a-zA-Z]")){exit};print}' "$PKGBUILD" > "$TMP";
+		bash -c "source $TMP; printf '%s\n' \${source[@]} | grep '^http'";
+		rm "$TMP";
+done
diff --git a/stack.yaml b/stack.yaml
new file mode 100644
index 0000000..a2a3cdc
--- /dev/null
+++ b/stack.yaml
@@ -0,0 +1,11 @@
+packages:
+- location:
+    git: https://git.yori.cc/https-everywhere-else/https-everywhere-rules.git
+    commit: ed1933f2c5e28a1faa844f1755a636d58683009b
+  extra-dep: true
+flags:
+  https-everywhere-rules:
+    build-examples: false
+extra-deps:
+- functor-infix-0.0.5
+resolver: lts-8.24