prepare tiles saving

osm/getTiles.py

@@ -5,10 +5,12 @@ import progressbar
 from pathlib import Path
 import pngquant
 from random import randrange
+import redis
 
-server = ['a','b','c']
+r = redis.Redis()
+server = "88.198.17.138"
 
-FOLDER = '/home/m/osm'
+HOME = '/mnt/osm'
 
 OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
 pngquant.config(min_quality=50, max_quality=80)
@@ -19,24 +21,32 @@ headers = {
 
 
 def base_url(zoom, x, y):        
-    BASE_URL = 'https://{SERVER}.tile.openstreetmap.org/{Z}/{X}/{Y}.png'
+    BASE_URL = 'http://{SERVER}/tile/{Z}/{X}/{Y}.png'
     return BASE_URL.format(Z=zoom, X=x, Y=y, SERVER=server[randrange(3)])
 
 def get_pngs(zoom):
+    progress = progressbar.ProgressBar()
     max_x = int(pow(2,zoom)-1)
-    for x in range(0, max_x):
-        print(str(x) + '/' + str(max_x))
-        progress = progressbar.ProgressBar()
-        for y in progress(range(0, int(pow(2,zoom)-1))):
-            Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
-                
+    for x in progress(range(0, max_x)):
+        if r.get("stop"):
+                break
+        r.set("x", x)   
+        for y in range(0, int(pow(2,zoom)-1)):
+            if r.get("stop"):
+                break
+            Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)  
             img_data = requests.get(base_url(zoom, x, y), headers=headers).content
             png = pngquant.quant_data(data=img_data)
             with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
                 handler.write(png[1])
+            r.set("y", y)
 
 if __name__ == '__main__':
-    zoom = int(sys.argv[1])
-    print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
-    get_pngs(zoom)
+    for zoom in range(0,10):
+        if r.get("stop"):
+                break
+        r.set('zoom', zoom)
+        print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
+        print("get zoom level: " + str(zoom))
+        get_pngs(zoom)
 

osm/getTilesThreaded.py

@@ -0,0 +1,90 @@
+from math import pow
+import requests
+import sys
+import progressbar
+from pathlib import Path
+import pngquant
+from random import randrange
+import threading
+import time
+import q
+import os
+import random
+import string
+
+def rnd_file():
+    # printing lowercase
+    letters = string.ascii_lowercase
+    return "/tmp/" + ( ''.join(random.choice(letters) for i in range(32)) ) + ".png"
+
+DEFAULT_WORKER = 5
+thread = []
+
+server = "88.198.17.138"
+
+HOME = '/mnt/osm'
+
+OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
+pngquant.config(min_quality=50, max_quality=80)
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0'
+}
+
+def base_url(zoom, x, y):        
+    BASE_URL = 'http://{SERVER}/tile/{Z}/{X}/{Y}.png'
+    return BASE_URL.format(Z=zoom, X=x, Y=y, SERVER=server[randrange(3)])
+
+def get_and_process_tile(x=None,zoom=None):
+    if None not in [x, zoom]:
+        pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
+        q("using " +  pngquant.config()["tmp_file"])
+        for y in range(0, int(pow(2,zoom)-1)):
+            q(str(x) + ',' + str(y) + ',' + str(zoom))
+            # Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
+                
+            # img_data = requests.get(base_url(zoom, x, y), headers=headers).content
+            # png = pngquant.quant_data(data=img_data)
+            # with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
+            #     handler.write(png[1])
+        try:
+            os.remove(pngquant.config()["tmp_file"])
+        except:
+            return
+    return
+
+
+def get_pngs(zoom):
+    max_x = int(pow(2,zoom)-1)
+    progress = progressbar.ProgressBar()
+    for x in progress(range(0, max_x)):
+        n = -1
+        while True:
+            n = n + 1
+            # check if a worker is done and update status                
+            if thread[n % DEFAULT_WORKER].isAlive() is False:
+                q("worker " + str(n % DEFAULT_WORKER) + " is free now")
+                break
+            else:
+                q("no free worker available ...")
+                time.sleep(2.0)
+
+        thread[n % DEFAULT_WORKER] = threading.Thread(target=get_and_process_tile, args=[x, zoom])
+        thread[n % DEFAULT_WORKER].start()
+        q("worker " + str(n % DEFAULT_WORKER) + " busy with " + str(x))
+
+
+        
+
+if __name__ == '__main__':
+    zoom = int(sys.argv[1])
+
+    # init 5 threads
+    for n in range(0, DEFAULT_WORKER):
+        thread.append(threading.Thread(target=get_and_process_tile, args=[None,None]))
+        thread[n].start()
+        print("spawning thread " + str(n+1) + "/" + str(DEFAULT_WORKER))
+
+    print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
+    get_pngs(zoom)
+