update osm

osm/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3-slim
+
+RUN apt update && apt install pngquant -y
+
+RUN pip install redis pngquant progressbar requests q
+
+COPY get* /

osm/getTiles.py

@@ -7,12 +7,12 @@ import pngquant
 from random import randrange
 import redis
 
-r = redis.Redis()
+r = redis.Redis(host="redis")
 server = "88.198.17.138"
 
-HOME = '/mnt/osm'
+HOME = '/mnt/osm/'
 
-OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
+OUTPUT = HOME + '{PNG}/{Z}/{X}/{Y}.png'
 pngquant.config(min_quality=50, max_quality=80)
 
 headers = {
@@ -34,15 +34,20 @@ def get_pngs(zoom):
         for y in range(0, int(pow(2,zoom))):
             if r.get("stop"):
                 break
-            Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)  
+            Path(HOME + 'optimized/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)  
+            Path(HOME + 'origin/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)  
             img_data = requests.get(base_url(zoom, x, y), headers=headers).content
             png = pngquant.quant_data(data=img_data)
-            with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
-                handler.write(png[1])
+            with open(OUTPUT.format(PNG='optimized',Z=zoom,X=x,Y=y), 'wb') as handler:
+                if len(png[1]) < len(img_data):
+                    handler.write(png[1])
+                else:
+                    handler.write(img_data)
+
             r.set("y", y)
 
 if __name__ == '__main__':
-    for zoom in range(0,10):
+    for zoom in range(0,9):
         if r.get("stop"):
                 break
         r.set('zoom', zoom)

osm/getTilesThreaded.py

@@ -3,7 +3,6 @@ import requests
 import sys
 import progressbar
 from pathlib import Path
-import pngquant
 from random import randrange
 import threading
 import time
@@ -21,10 +20,10 @@ thread = []
 
 server = "88.198.17.138"
 
-HOME = '/mnt/osm'
+HOME = '/mnt/osm/optimized'
 
 OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
-pngquant.config(min_quality=50, max_quality=80)
+
 
 headers = {
     'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0'
@@ -36,20 +35,44 @@ def base_url(zoom, x, y):
 
 def get_and_process_tile(x=None,zoom=None):
     if None not in [x, zoom]:
-        pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
-        q("using " +  pngquant.config()["tmp_file"])
+        import pngquant
+        N = 0
+        while True:
+            try:
+                pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
+                break
+            except:
+                N += 1
+                if N > 10:
+                    break
+                continue
+
         for y in range(0, int(pow(2,zoom))):
             q(str(x) + ',' + str(y) + ',' + str(zoom))
-            # Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
+            Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
                 
-            # img_data = requests.get(base_url(zoom, x, y), headers=headers).content
-            # png = pngquant.quant_data(data=img_data)
-            # with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
-            #     handler.write(png[1])
+            data = requests.get(base_url(zoom, x, y), headers=headers).content
+            N = 0
+            while True:
+               try: 
+                  png = pngquant.quant_data(data=data)
+                  break
+               except:
+                   N += 1
+                   if N > 10:
+                       break
+                   continue
+
+            with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
+                if len(png[1]) < len(data):
+                    handler.write(png[1])
+                else:
+                    handler.write(data)
         try:
             os.remove(pngquant.config()["tmp_file"])
+            q("remove successfull")
         except:
-            return
+            q("remove not possible: " + str(pngquant.config()["tmp_file"]))
     return
 
 
@@ -61,7 +84,7 @@ def get_pngs(zoom):
         while True:
             n = n + 1
             # check if a worker is done and update status                
-            if thread[n % DEFAULT_WORKER].isAlive() is False:
+            if thread[n % DEFAULT_WORKER].is_alive() is False:
                 q("worker " + str(n % DEFAULT_WORKER) + " is free now")
                 break
             else: