Make sure all extensions are covered

scripts/build

@@ -251,9 +251,15 @@ def generate_ftdetect
 
   EOS
 
-  ambiguous_extensions = languages.flat_map { |_, v| v["extensions"] || [] }
-    .group_by(&:itself).transform_values(&:count)
-    .select { |a, b| b > 1 }.keys.map { |a| a[1..] }.to_set
+  ambiguous_extensions = PACKAGES.flat_map { |e| e["filetypes"] }.flat_map do |e|
+    if e["linguist"]
+      ((languages[e["linguist"]]["extensions"] || []).map { |e| e[1..-1] } | e.fetch("extra_extensions", [])) - e.fetch("ignored_extensions", [])
+    else
+      e.fetch("extensions", [])
+    end
+  end.group_by(&:itself).transform_values(&:count).select { |a, b| b > 1 }.keys.to_set
+
+  expected_filetypes = detect_filetypes
 
   for package in PACKAGES
     name = package.fetch("name")
@@ -266,8 +272,9 @@ def generate_ftdetect
     filetypes = package["filetypes"] or raise "Unknown filetype for: #{package["name"]}"
 
     for filetype in filetypes
+      name = filetype.fetch("name")
       syntax = filetype["syntax"] ? " syntax=#{filetype["syntax"]}" : ""
-      set_command = package.fetch("custom_set", "set ft=#{filetype.fetch("name")}#{syntax}")
+      set_command = package.fetch("custom_set", "set ft=#{name}#{syntax}")
 
       linguist = filetype["linguist"] ? languages.fetch(filetype["linguist"]) : {}
       extensions = filetype["extensions"] || linguist.fetch("extensions", []).map { |e| e[1..] }
@@ -276,6 +283,16 @@ def generate_ftdetect
       filenames = filetype["filenames"] || linguist.fetch("filenames", [])
       filenames = (filenames | filetype.fetch("extra_filenames", [])) - filetype.fetch("ignored_filenames", [])
 
+      if expected_filetypes[name] && !filetype["syntax"]
+        for e in expected_filetypes.fetch(name)[:extensions] - extensions - expand_all(filetype.fetch("ignored_extensions", []))
+          puts "Probable missing extension for #{name}: #{e}"
+        end
+
+        for e in expand_all(expected_filetypes.fetch(name)[:filenames]) - expand_all(filenames).flat_map { |e| [e, e.gsub(/^\./, '')] } - expand_all(filetype.fetch("ignored_filenames", [])) - ['*']
+          puts "Probable missing filename for #{name}: #{e}"
+        end
+      end
+
       for extension in extensions.sort
         outer_filetype = filetype["outer_filetype"]
         if outer_filetype
@@ -306,6 +323,12 @@ def generate_ftdetect
     let &cpo = s:cpo_save
     unlet s:cpo_save
   EOS
+ 
+  # filetypes = detect_filetypes
+  #
+  # for filetype in filetypes
+  #   
+  # end
 
   File.write('ftdetect/polyglot.vim', output)
 end
@@ -333,11 +356,86 @@ def generate_tests
   end
 
   File.write('scripts/test_filetypes.vim', output)
+
+
 end
 
-download
-extract
+def brace_expansion(s)
+  r=1                                       # Dummy value to forward-declare the parse function `r`
+  t=->x{                                    # Function to parse a bracket block
+    x=x[0].gsub(/^{(.*)}$/){$1}             # Remove outer brackets if both are present
+                                            # x[0] is required because of quirks in the `scan` function
+    x=x.scan(/(({(\g<1>|,)*}|[^,{}]|(?<=,|^)(?=,|$))+)/)
+                                            # Regex black magic: collect elements of outer bracket
+    x.map{|i|i=i[0];i[?{]?r[i]:i}.flatten   # For each element with brackets, run parse function
+  }
+  r=->x{                                    # Function to parse bracket expansions a{b,c}{d,e}
+    i=x.scan(/({(\g<1>)*}|[^{} ]+)/)        # Regex black magic: scan for adjacent sets of brackets
+    i=i.map(&t)                             # Map all elements against the bracket parser function `t`
+    i.shift.product(*i).map &:join          # Combine the adjacent sets with cartesian product and join them together
+  }
+  s.split.map(&r).flatten
+end
+
+def square_expansion(s)
+  return [s] unless s.include?('[')
+  s.scan(/(\[[^\]]+\]|[^\[]+)/).map { |x| x[0] }
+    .map { |x| x[0] == "[" ? x[1..-2].split("") : [x] }
+    .reduce(&:product).map(&:flatten).map(&:join)
+end
+
+def comma_expanson(s)
+  s.scan(/{[^{]+}|[^{]+/).map { |a| a[0] == "{" ? a : a.split(",", -1) }.reduce([]) do |a, b|
+    a.size > 0 ?
+      (b.is_a?(String) ?
+         a[0..-2] + [a[-1] + b] :
+         a[0..-2] + [a[-1] + b[0]] + b[1..-1]) :
+      [b].flatten
+  end
+end
+
+def expand_all(pattern)
+  if pattern.is_a?(Array)
+    return pattern.flat_map { |p| expand_all(p) }
+  end
+
+  comma_expanson(pattern).flat_map do |e|
+    brace_expansion(e).flat_map do |e2|
+      square_expansion(e2)
+    end
+  end
+end
+
+def detect_filetypes
+  filetypes = Dir['tmp/*/ftdetect/*.vim'].flat_map do |file|
+    contents = File.read(file).gsub(/^\s*au(tocmd)?!?\s*$/, '')
+    results = contents.scan(/^\s*(?:au!|au|au[^g][^ ]*) +(?:\S+)\s+(\S+)[\s\\]+([^\n]+)/)
+    results = results.map do |a, b|
+      [
+        a,
+        b.gsub(/call (?:s:setf|s:StarSetf)\('([^']+)'\)/i, 'setf \1')
+          .gsub(/set(?:local)?\s+(?:ft|filetype)=(\S+)/, 'setf \1')
+        .gsub(/setf\S*/, 'setf')
+        .gsub(/.*setf\s+(\S+).*/, 'setf \1')
+      ]
+    end.select { |a, b| b.match(/setf \S+/) }.map { |a, b| [a, b.split(" ")[1]] }
+    results
+  end
+
+  Hash[filetypes.flat_map do |ext, filetype|
+    expand_all(ext).map { |e| [filetype, e] }
+  end.group_by { |a, b| a }.map { |a, b| [a, b.map { |c, d| d }] }.map { |a, b|
+    [a, {
+      extensions: b.select { |x| x.match(/^\*\.[^\/]+$/) }.map { |a| a.strip[2..] },
+      filenames: b.select { |x| !x.match(/^\*\.[^\/]+$/) }
+    }]
+  }]
+end
+
+# download
+# extract
 generate_ftdetect
-generate_tests
-puts(" Bye! Have a wonderful time!")
-FileUtils.rm_rf("tmp")
+# generate_tests
+# puts(" Bye! Have a wonderful time!")
+# FileUtils.rm_rf("tmp")
+#puts(comma_expanson("*.clj,*.cljs,*.edn,*.cljx,*.cljc,{build,profile}.boot"))