Первоначальный импорт

7dc41cfd · Иван Бегтин · 7dc41cfd · 7dc41cfd · 7dc41cfd · 7dc41cfd
Commit 7dc41cfd authored May 28, 2019 by Иван Бегтин
51 changed files
--- a/LICENSE
+++ b/LICENSE
+The MIT License (MIT)
+
+Copyright (c) 2016 NGO "Informational Culture"
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/README.md
+++ b/README.md
+# blindprocurement
+Слепые закупки. Код алгоритмов выявления искажения анонсов госзакупок так чтобы их не могли находить "чужие" поставщики
--- a/all_top_procedures_16022013.xls
+++ b/all_top_procedures_16022013.xls
--- a/dashed-txt.tsv
+++ b/dashed-txt.tsv
--- a/dashed-txt.xls
+++ b/dashed-txt.xls
--- a/dashed.txt
+++ b/dashed.txt
--- a/docs/opendata_blindprocurement.pdf
+++ b/docs/opendata_blindprocurement.pdf
--- a/export/final.xlsx
+++ b/export/final.xlsx
--- a/export/final_export.xlsx
+++ b/export/final_export.xlsx
--- a/export/full.tsv
+++ b/export/full.tsv
--- a/export/level1.tsv
+++ b/export/level1.tsv
--- a/export/level1.xls
+++ b/export/level1.xls
--- a/export/level2.xls
+++ b/export/level2.xls
--- a/export/level3.xls
+++ b/export/level3.xls
--- a/export/orgs.csv
+++ b/export/orgs.csv
+   	172
+   "   ."	77
+    " -     . . "       	56
+     "  - "	47
+    ""	37
+     "     " "     "	36
+   	32
+    ""	22
+     	18
+     "  """"	18
+     "   "	17
+     	16
+            -   ""	15
+   	15
+        "      "	13
+     "     "	13
+             - -     ""	13
+       	13
+ " "	12
+          	12
+     "-     ""	12
+          4  	12
+   "5           ()"	11
+     "-       " "	11
+   	11
+   " -   "	10
+     -   	9
+     "   - "	9
+   " "   	9
+     	9
+      "    76"	9
+     "      "    	9
+     " "	8
+       	8
+    "   2"	8
+               -	8
+     ( )  .       ()	8
+  	8
+        	8
+    "      "	8
+    "     "	7
+        	7
+      	7
+   	7
+     "  "	7
+    	7
+    "-  "	7
+      	7
+  	6
+   	6
+     " "	6
+   -   	6
+       N 62	6
+     "      "	6
+     "   . "	6
+         . -	5
+     " -  -     "	5
+   "  ()   1"	5
+   "       "	5
+    	5
+    -  -  ,    , " "    	5
+       "  "	5
+    " -    ""	5
+        	5
+      ""	4
+   	4
+       N72	4
+  "       "	4
+   " "	4
+       	4
+    " -      .. "       	4
+   ""	4
+     	4
+      " -          "	4
+     ""	4
+     	4
+    -  ""	4
+        	4
+   "   25        "	4
+ ,      	3
+-     " - "	3
+            	3
+   	3
+       N 73	3
+        	3
+    "-  22    "	3
+     	3
+          ""  	3
+        ,          	3
+       53	3
+ -    	3
+     	3
+   "  ()  "	3
+         	3
+   "   "  	3
+     () "   4"	3
+        	3
+   "      "	3
+        -      1 ""	3
+   "    "      	3
+       	3
+        	3
+    -    	3
+   "    "   " "	3
+          . ..    	3
+      	3
+    "     " "     "	3
+   	3
+        "   "	3
+   "   1        -"	3
+   "     "	3
+     -   	3
+-     " - - - ""	3
+       N 41	3
+   "  "	3
+   " "	3
+    	3
+              -       3 ""	3
+     -    	3
+      "    "	2
+       " "	2
+    "  -        "            	2
+-    ()    "-   -      "	2
+      "      ""	2
+  	2
+   "   14       -   - "	2
+   "  "    	2
+       N 6	2
+       N 9	2
+    " "	2
+    	2
+   	2
+            37 ""	2
+     "        "	2
+  -  "          "	2
+    "   "    	2
+   "  "	2
+              	2
+        	2
+    "   "       	2
+    "      .. "    	2
+   (  )     	2
+   " -   "	2
+    "        "	2
+         "         "	2
+      "           "	2
+   "   - "	2
+    -     7  	2
+     -   	2
+     	2
+         "  "	2
+        5      	2
+     "  78    "	2
+   "   8         "	2
+    	2
+   "    "	2
+   ( )   ()	2
+             	2
+   " "	2
+   	2
+     	2
+       N 77	2
+      	2
+   "   6        "	2
+       	2
+    ()    	2
+     "   "	2
+    "  35533"	2
+        	2
+      "  "	2
+         ""	2
+        	2
+    	2
+    "- ""       "	2
+      -    	2
+        	2
+         	2
+        	2
+   "  "	2
+    "  N 30 ""	2
+     "   "	2
+      "  "	2
+     () "     ()"	2
+       79	2
+  -          	2
+    "."	2
+               	2
+   	2
+   " -"	2
+   	2
+   "   "	2
+  	2
+    "   "	2
+   "  "	2
+     "  "	2
+   	2
+    "  " "	2
+       	2
+          56	2
+        	2
+     	2
+        144      -	2
+-       	2
+    " "	2
+  	2
+      	1
+       	1
+   	1
+    "-"	1
+    "7       "	1
+     	1
+     	1
+     "   "	1
+        "      (. )"	1
+           	1
+  -       .   	1
+   "   1         "	1
+      	1
+       "      "	1
+    "    ,       "	1
+    "   "   " "	1
+ ()      -    "     "	1
+       "   35"	1
+     ""    	1
+    "         "	1
+        22     -	1
+    -     10 .	1
+         "  -      .. "    	1
+         " -      "	1
+    "   4"	1
+        	1
+     "     "	1
+        	1
+        	1
+    "   41  --"	1
+     "  -    -  "	1
+        	1
+   "   10        "	1
+          	1
+        	1
+          	1
+      "   "	1
+       " - "	1
+  -        - -   - 	1
+     "        "	1
+  44231	1
+   "-  -    ""	1
+   "    "  	1
+   "   3        "	1
+   "   "	1
+    "-         "	1
+       ""       	1
+        	1
+           	1
+     ( )     	1
+        	1
+     -   	1
+    " "	1
+    "-- "	1
+     	1
+   " -     "	1
+       "   426  "	1
+        13 ""    2 ""	1
+    "      "	1
+   "   "	1
+   "  ""	1
+     	1
+      38  	1
+-        "    3"	1
+   "             "	1
+    " -    "       	1
+    	1
+   "      "     ()	1
+       	1
+   "    N 1"	1
+    	1
+          " -    ()"	1
+ -     --	1
+   "-   - "	1
+      	1
+       	1
+  3701	1
+   "   6         "	1
+    " - "	1
+      	1
+   	1
+        	1
+     "   "	1
+    " -  - "	1
+  -  "    1        "	1
+       " -   ,          ""	1
+            	1
+  	1
+           	1
+        	1
+   "    ,          ""	1
+    	1
+       ""	1
+ -    	1
+        	1
+  -       -   	1
+      -    "       "	1
+     . 	1
+        3	1
+     	1
+           - 	1
+  "        "	1
+   "    4        "	1
+       	1
+   	1
+    "- "        	1
+        	1
+   " "   	1
+     ""	1
+   "  " -   	1
+   "  "	1
+   "     "	1
+    "            . .. "	1
+        	1
+   "  "    	1
+        "      "	1
+  	1
+        13      	1
+         "  "	1
+       (   )      	1
+      	1
+      "   15    "	1
+   	1
+   "      " "	1
+     	1
+     "   49   -    "	1
+        -    9      	1
+       " "	1
+    "  71330"	1
+        "      "	1
+         	1
+   "   5        "	1
+     "   -   ""	1
+  	1
+   "-  " "	1
+    "    1"	1
+      "   N9"	1
+     	1
+    "  45187"	1
+         	1
+      76    --	1
+     	1
+   "  "  	1
+   	1
+       	1
+     	1
+        -    5      	1
+    " "	1
+   "       " "	1
+         	1
+       	1
+      	1
+        27     -	1
+   -       	1
+      "-  51    "	1
+  	1
+-     "  15"	1
+    	1
+     	1
+      "     .. "	1
+      10   	1
+     ( )     	1
+    "  "     	1
+   "-        . -   "	1
+      	1
+    "     "	1
+        112     -	1
+    "   "	1
+   "-         "	1
+   "   2         "	1
+         " -       ""	1
+    -   	1
+    "    3 . --"     	1
+     	1
+     "       "	1
+   "-      - "	1
+          "-   "	1
+   "  ()   2"	1
+    "  "	1
+      	1
+   "  ()   6"	1
+   "   3        "	1
+     "  "	1
+    "        "	1
+      - "	1
+     	1
+          34	1
+ -      .   	1
+   	1
+  -          	1
+          	1
+  -          	1
+       "     - "	1
+              	1
+    -  . ..  	1
+       "    ""	1
+        "  "	1
+    	1
+         	1
+       "   2" . 	1
+         " -      33"	1
+    "-   ,         "	1
+    ,      	1
+    "        "	1
+    " - "    	1
+    "   "     	1
+         	1
+    "         "  - 	1
+   "-          "	1
+            	1
+ " "  	1
+    "        - "	1
+    "      "	1
+-     "    68"	1
+            	1
+   	1
+    "4  - "    	1
+     "  "	1
+     	1
+       "   -"       	1
+-        "    22"	1
+     	1
+     " "	1
+  	1
+    "   "	1
+   " - "     	1
+              -    ""	1
+     -      ,  ()         ""  	1
+ " "  	1
+         "     128"	1
+       -  ,    ,   3 .	1
+        "-  "-"	1
+     "      "	1
+          	1
+        	1
+   	1
+   "   "   " "  	1
+   "     "	1
+  	1
+     "     2"  . 	1
+        	1
+    "   "	1
+      "-    . ..     "	1
+        	1
+   ()    ,       ()   (VIII )  18   -	1
+     "       "	1
+       "   "	1
+    "   " "  - 	1
+   "     28"	1
+     ( )        	1
+-        	1
+     "  . .."	1
+         62 ""	1
+   	1
+     	1
+        	1
+        	1
+      	1
+    "-      "	1
+   "          "	1
+  "-  ""	1
+      "    51    "	1
+     "       "	1
+        	1
+              -     20 ""	1
+       " "	1
+        411 ""        -	1
+    " " "     "	1
+     -      50 "" .   	1
+        13      -     -	1
+   " "	1
+       	1
+  -          	1
+    "-97"	1
+   "-           "	1
+  -          	1
+     N 82	1
+        	1
+     	1
+       "-      ""	1
+  ,     	1
+      	1
+   	1
+      N 7	1
+     "  -  "-"	1
+        3	1
+      5   	1
+    "  -    "    	1
+    "  54729"	1
+     	1
+      "   7  -"	1
+        	1
+   "   ,      "	1
+    "      "	1
+  "        "	1
+      3   	1
+   "   "	1
+         ( .)	1
+    	1
+            	1
+   "     "	1
+      "    1    "	1
+           	1
+   	1
+             	1
+     ( )      - 	1
+  -   -  "  "	1
+       " "	1
+   ""	1
+      "   2"	1
+      "   3  "	1
+   "   "	1
+    	1
+       "-       ""	1
+     	1
+     -    75	1
+   "  ()  "	1
+     	1
+  -   "  "	1
+         	1
+    "   5 ." .   ..	1
+        	1
+    -    	1
+   "-  """"   	1
+     -   	1
+     "        " "	1
+  	1
+    "        N2  "	1
+     " - "	1
+    "   N9  "	1
+        	1
+    "  "	1
+    "    32"  	1
+       " "	1
+        4	1
+           - ""	1
+-    "         , ,  -"	1
+        	1
+    " "	1
+      	1
+   "   "       	1
+      9   	1
+     "      "	1
+   "       -    "	1
+    " "	1
+  ""	1
+        -  	1
+-        "  "  	1
+        2      	1
+  -        "" (               )	1
+        11        	1
+    "  43753"	1
+     	1
+      	1
+    	1
+   " "     ()	1
+        	1
+     	1
+      "   -     " "	1
+   	1
+    "  "	1
+      "     4"	1
+   ,   	1
+ -       	1
+   " -   .. "	1
--- a/export/reduce.py
+++ b/export/reduce.py
+#!/usr/bin/env python
+# coding: utf8
+__author__ = 'ibegtin'
+
+import sys, os.path
+import locale
+from Levenshtein import distance
+import csv
+locale.setlocale(locale.LC_ALL, 'ru_RU')
+from pymongo import Connection
+import bson
+
+
+URLPAT = 'http://zakupki.gov.ru/pgz/public/action/orders/info/common_info/show?notificationId=%s'
+
+
+class OrgProcessor:
+    def __init__(self):
+        self.conn = Connection()
+        self.db = self.conn['gz']
+        self.coll = self.db['orgs']
+        self.coll.ensure_index('regNumber', 1)
+
+    def import_orgs(self):
+        reader = csv.DictReader(open('../allorgs.csv', 'r'), delimiter='\t')
+        n = 0
+        for row in reader:
+            id = row['regNumber']
+            if self.coll.find_one({'regNumber' : id}) is None:
+                try:
+                    self.coll.save(row)
+                except bson.errors.InvalidDocument:
+                    print 'Error', row.keys()
+            n += 1
+            if n % 1000 == 0: print n
+        print n
+
+    def merge(self):
+        reader = csv.DictReader(open('rules.tsv', 'r'), delimiter='\t')
+        fieldnames = reader.fieldnames
+        addfields = ['okogu_name', 'subordinationType_description', 'factualAddress_region_fullName', 'headAgency_fullName']
+        fieldnames.extend(addfields)
+        writer = csv.DictWriter(open('rules_final.tsv', 'w'), fieldnames=fieldnames, delimiter='\t')
+        writer.writeheader()
+        for row in reader:
+            row['customer_regnum'] = "0" + row['customer_regnum']
+            item =  self.coll.find_one({'regNumber' : row['customer_regnum']})
+            if item is None:
+                continue
+            for k in row.keys():
+                row[k] = row[k].decode('utf8') if row[k] is not None else ""
+            for field in addfields:
+                row[field] = item[field]
+            for k in row.keys():
+                row[k] = row[k].encode('windows-1251') if row[k] is not None else ""
+            writer.writerow(row)
+        pass
+
+
+def extract(filename):
+    orgs = {}
+    regions = {}
+    reader = csv.DictReader(open(filename, 'r'), delimiter='\t')
+    for row in reader:
+        region = regions.get(row['factualAddress_region_fullName'].strip(), 0)
+        region += 1
+        regions[row['factualAddress_region_fullName'].strip()] = region
+
+        region = orgs.get(row['orgname'].strip(), 0)
+        region += 1
+        orgs[row['orgname'].strip()] = region
+
+    f = open('orgs.csv', 'w')
+    thedict = sorted(orgs.items(), lambda x, y: cmp(x[1], y[1]), reverse=True)
+    for k, v in thedict:
+        s = '%s\t%d' %(k.decode('utf8'), v) + '\n'
+        f.write(s.encode('windows-1251'))
+
+    f = open('regions.csv', 'w')
+    thedict = sorted(regions.items(), lambda x, y: cmp(x[1], y[1]), reverse=True)
+    for k, v in thedict:
+        s = '%s\t%d' %(k.decode('utf8'), v) + '\n'
+        f.write(s.encode('windows-1251'))
+
+
+#print row
+
+if __name__ == "__main__":
+    proc = OrgProcessor()
+#    proc.import_orgs()
+    proc.merge()
+#    extract(sys.argv[1])
+
--- a/export/regions.csv
+++ b/export/regions.csv
+ 	321
+-	236
+ 	218
+	83
+ 	79
+ 	70
+ 	68
+ 	60
+ 	42
+ 	31
+ // 	24
+ 	23
+ 	22
+ 	19
+ 	18
+  	18
+-   -  	17
+ 	16
+ 	16
+ 	15
+ 	15
+ 	13
+ 	13
+ 	13
+ 	12
+ 	11
+ 	10
+  - 	10
+ 	10
+ 	10
+ 	10
+ 	9
+ 	9
+ 	8
+ 	8
+ 	8
+ 	8
+ 	8
+ 	7
+ 	6
+ 	6
+ 	6
+ 	5
+ 	5
+ 	5
+ 	5
+ 	4
+ 	4
+ 	4
+ 	4
+ 	4
+ 	4
+ 	3
+ 	3