updater & find_redundant: rewrite in ruby

autoupdater.rb

@@ -32,7 +32,7 @@ ENV.fetch("JOBS", "1").to_i.times.each do
 
             if v.check_domain_verbose(domain, enable_cdnlist: false, show_green: true)
                 Filelock '.git.lock' do
-                    puts `./updater.py -a #{domain}`
+                    puts `./updater.rb -a #{domain}`
                     puts `git commit -S -am "accelerated-domains: add #{domain}"` if $?.success?
                     puts `./update-local` if $?.success?
                 end

find_redundant.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-''' Find redundant items in accelerated-domains.china.conf.
-    e.g. 'bar.foo.com' is redundant for 'foo.com'.
-'''
-
-
-from collections.abc import Iterable
-
-
-def load(conf_file):
-    ''' Parse conf file & Prepare data structure
-        Returns: [ ['abc', 'com'],
-                   ['bar', 'foo', 'com'],
-                   ... ]
-    '''
-
-    results = []
-    if isinstance(conf_file, str):
-        lines = open(conf_file, 'r').readlines()
-    elif isinstance(conf_file, Iterable):
-        lines = iter(conf_file)
-    for line in lines:
-        line = line.strip()
-        if line == '' or line.startswith('#'):
-            continue
-        # A domain name is case-insensitive and
-        # consists of several labels, separated by a full stop
-        domain_name = line.split('/')[1]
-        domain_name = domain_name.lower()
-        domain_labels = domain_name.split('.')
-        results.append(domain_labels)
-
-    # Sort results by domain labels' length
-    results.sort(key=len)
-    return results
-
-
-def find(labelses):
-    ''' Find redundant items by a tree of top-level domain label to sub-level.
-        `tree` is like { 'com': { 'foo: { 'bar': LEAF },
-                                  'abc': LEAF },
-                         'org': ... }
-    '''
-    redundant = []
-    tree = {}
-    LEAF = 1
-    for labels in labelses:
-        domain = '.'.join(labels)
-        # Init root node as current node
-        node = tree
-        while len(labels) > 0:
-            label = labels.pop()
-            if label in node:
-                # If child node is a LEAF node,
-                # current domain must be an existed domain or a subdomain of an existed.
-                if node[label] == LEAF:
-                    print(f"Redundant found: {domain} at {'.'.join(labels)}")
-                    redundant.append(domain)
-                    break
-            else:
-                # Create a leaf node if current label is last one
-                if len(labels) == 0:
-                    node[label] = LEAF
-                # Create a branch node
-                else:
-                    node[label] = {}
-            # Iterate to child node
-            node = node[label]
-    return redundant
-
-
-def find_redundant(conf_file):
-    return find(load(conf_file))
-
-if __name__ == '__main__':
-    find_redundant('accelerated-domains.china.conf')

find_redundant.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/ruby
+
+""" Find redundant items in accelerated-domains.china.conf.
+    e.g. 'bar.foo.com' is redundant for 'foo.com'.
+"""
+
+def load(conf_file)
+    """ Parse conf file & Prepare data structure
+        Returns: [ ['abc', 'com'],
+                   ['bar', 'foo', 'com'],
+                   ... ]
+    """
+
+    results = []
+    if conf_file.is_a? String
+        lines = File.readlines(conf_file)
+    elsif conf_file.is_a? Array
+        lines = conf_file
+    end
+    lines.map do |line|
+        line = line.chomp
+        next if line.empty? or line.start_with?('#')
+        # A domain name is case-insensitive and
+        # consists of several labels, separated by a full stop
+        domain_name = line.split('/')[1]
+        domain_name = domain_name.downcase
+        domain_labels = domain_name.split('.')
+        results << domain_labels
+    end
+
+    # Sort results by domain labels' length
+    results.sort_by(&:length)
+end
+
+
+LEAF = 1
+def find(labelses)
+    """ Find redundant items by a tree of top-level domain label to sub-level.
+        `tree` is like { 'com': { 'foo: { 'bar': LEAF },
+                                  'abc': LEAF },
+                         'org': ... }
+    """
+    redundant = []
+    tree = {}
+    labelses.each do |labels|
+        domain = labels.join('.')
+        # Init root node as current node
+        node = tree
+        until labels.empty?
+            label = labels.pop
+            if node.include? label
+                # If child node is a LEAF node,
+                # current domain must be an existed domain or a subdomain of an existed.
+                if node[label] == LEAF
+                    puts "Redundant found: #{domain} at #{labels.join('.')}"
+                    redundant << domain
+                    break
+                end
+            else
+                # Create a leaf node if current label is last one
+                if labels.empty?
+                    node[label] = LEAF
+                # Create a branch node
+                else
+                    node[label] = {}
+                end
+            end
+            # Iterate to child node
+            node = node[label]
+        end
+    end
+    redundant
+end
+
+def find_redundant(conf_file)
+    return find(load(conf_file))
+end
+
+if __FILE__ == $0
+    find_redundant('accelerated-domains.china.conf')
+end

updater.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-from __future__ import unicode_literals
-from argparse import ArgumentParser
-import idna
-import sys
-import find_redundant
-
-if __name__ == "__main__":
-    parser = ArgumentParser(description="dnsmasq-china-list updater")
-    parser.add_argument(
-        '-a', '--add',
-        metavar="DOMAIN",
-        nargs="+",
-        help='Add one or more new domain(s) (implies -s)',
-    )
-    parser.add_argument(
-        '-d', '--delete',
-        metavar="DOMAIN",
-        nargs="+",
-        default=[],
-        help='Remove one or more old domain(s) (implies -s)',
-    )
-    parser.add_argument(
-        '-s', '--sort',
-        action='store_true',
-        default=True,
-        help='Sort the list (default action)',
-    )
-    parser.add_argument(
-        '-f', '--file',
-        nargs=1,
-        default=["accelerated-domains.china.conf"],
-        help="Specify the file to update (accelerated-domains.china.conf by default)",
-    )
-
-    options = parser.parse_args()
-
-    with open(options.file[0]) as f:
-        lines = list(f)
-
-    changed = False
-
-    if options.add:
-        options.sort = True
-
-        for domain in options.add:
-            encoded_domain = idna.encode(domain).decode()
-            new_line = f"server=/{encoded_domain}/114.114.114.114\n"
-            disabled_line = f"#server=/{encoded_domain}/114.114.114.114"
-            if new_line in lines:
-                print(f"Domain already exists: {domain}")
-            else:
-                for line in lines:
-                    if line.startswith(disabled_line):
-                        print(f"Domain already disabled: {domain}")
-                        break
-                else:
-                    print(f"New domain added: {domain}")
-                    lines.append(new_line)
-                    changed = True
-
-    options.delete += find_redundant.find_redundant(lines)
-
-    if options.delete:
-        options.sort = True
-
-        for domain in options.delete:
-            target_line = f"server=/{idna.encode(domain).decode()}/114.114.114.114\n"
-            if target_line not in lines:
-                print(f"Failed to remove domain {domain}: not found.")
-            else:
-                print(f"Domain removed: {domain}")
-                lines.remove(target_line)
-                changed = True
-
-    if (options.add or options.delete) and not changed:
-        sys.exit(1)
-
-    if options.sort:
-        lines.sort(key=lambda x: x.lstrip("#"))
-
-    with open(options.file[0], "w") as f:
-        f.write(''.join(filter(lambda line: line.strip(), lines)))

updater.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/ruby
+require 'domain_name'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.sort = true
+options.file = "accelerated-domains.china.conf"
+options.add = []
+options.delete = []
+OptionParser.new do |opts|
+    opts.banner = "dnsmasq-china-list updater"
+
+    opts.on("-s", "--[no-]sort", "Sort the list (default action)") do |s|
+        options.sort = s
+    end
+
+    opts.on("-f", "--file FILE", "Specify the file to update (accelerated-domains.china.conf by default)") do |f|
+        options.file = f
+    end
+
+    opts.on("-a", "--add domain1,domain2", Array, "Add domain(s) to the list (implies -s)") do |a|
+        options.add = a
+        options.sort = true
+    end
+
+    opts.on("-d", "--delete domain1,domain2", Array, "Remove domain(s) from the list (implies -s)") do |d|
+        options.delete = d
+        options.sort = true
+    end
+end.parse!
+
+lines = File.readlines(options.file).filter { |line| !line.empty? }
+disabled_lines = lines.filter { |line| line.start_with?("#") }
+
+changed = false
+
+options.add.each do |domain|
+    domain = DomainName.normalize(domain)
+    new_line = "server=/#{domain}/114.114.114.114\n"
+    disabled_line = "#server=/#{domain}/114.114.114.114"
+    if lines.include? new_line
+        puts "Domain already exists: #{domain}"
+    else
+        if disabled_lines.any? { |line| line.start_with? disabled_line }
+            puts "Domain already disabled: #{domain}"
+        else
+            # Check for duplicates
+            test_domain = domain
+            while test_domain.include? '.'
+                test_domain = test_domain.partition('.').last
+                _new_line = "server=/#{test_domain}/114.114.114.114\n"
+                _disabled_line = "#server=/#{test_domain}/114.114.114.114"
+                if lines.include? _new_line 
+                    puts "Redundant domain already exists: #{test_domain}"
+                    break
+                elsif disabled_lines.any? { |line| line.start_with? _disabled_line }
+                    puts "Redundant domain already disabled: #{test_domain}"
+                    break
+                end
+            end
+            next if test_domain.include? '.'
+
+            puts "New domain added: #{domain}"
+            lines << new_line
+            changed = true
+        end
+    end
+end
+
+options.delete.each do |domain|
+    domain = DomainName.normalize(domain)
+    target_line = "server=/#{domain}/114.114.114.114\n"
+    unless lines.include? target_line
+        puts "Failed to remove domain #{domain}: not found."
+    else
+        puts "Domain removed: #{domain}"
+        lines.delete(target_line)
+        changed = true
+    end
+end
+
+fail "No changes made." if (options.add.length || options.delete.length) && !changed
+
+lines.sort_by! { |x| x.delete_prefix("#") } if options.sort
+
+File.write(options.file, lines.join)