Caching

db-read() -> 100ms

if request in cache,
return cache[request]

cashe is a hashtable
basic cache algorithm is below

import time

def complex_computation(a, b):
    time.sleep(.5)
    return a + b

cache = {}
def cached_computation(a, b):
    key = (a, b)
    if key in cache:
        r = cache[key]
    else:
        r = complex_computation(a, b)
        cache[key] = r
    return r

start_time = time.time()
print cached_computation(5, 3)
print "the first computation took %f second" % (time.time() - start_time)

when db query can be cashed in serverside as dictionary like below.

CACHE = {}
def top_arts();
        key = 'top'
        if key in CACHE:
            arts = CHACHE[key]
        else:
            loggin.error("DB QUERY")
            art = db.GqlQuery("SELECT * From Art "
                            "WHERE ancestor is :1"
                            "ORDER BY DESC")

        arts = list(arts)
        CACHE[key] = arts
    return arts

when posted, cache cleared.

CACHE = {}
def top_arts(update = False);
        key = 'top'
        if not update and key in CACHE:
            arts = CHACHE[key]
        else:
            loggin.error("DB QUERY")
            art = db.GqlQuery("SELECT * From Art "
                            "WHERE ancestor is :1"
                            "ORDER BY DESC")

        arts = list(arts)
        CACHE[key] = arts
    return arts

DB read should be only submission.
loadbalancer also can handle large traffic.

load balancere

n = -1
def get_server():
    global n
    n += 1
    return SERVERS[n % len(SERVERS)]

handler, url mapping DBmodel

parsing

>>> import urllib2
>>> import urllib
>>> p = urllib2.urlopen("http://www.google.com")
>>> p
>
>>> c = p.read()
>>> dir(p)
['__doc__', '__init__', '__iter__', '__module__', '__repr__', 'close', 'code', 'fileno', 'fp', 'getcode', 'geturl', 'headers', 'info', 'msg', 'next', 'read', 'readline', 'readlines', 'url']
>>> p.url
'http://www.google.co.jp/?gfe_rd=cr&ei=MtZkWNDXDYSL8QeD46cY'
>>> p.headers

>>> p.headers.items()
[('x-xss-protection', '1; mode=block'), ('set-cookie', 'NID=93=O65u9flBWzM92U9MzcezfIXaeG9itO-ala3ogt6T7fipovY5ily4QBNUxbUbsVga_hYeJEKWDq891mFaPgZm2Ya_1gvUZm37K2pNfFpOUVxCptVtOSAn3OXvUHCzKBaC; expires=Fri, 30-Jun-2017 09:24:02 GMT; path=/; domain=.google.co.jp; HttpOnly'), ('accept-ranges', 'none'), ('expires', '-1'), ('vary', 'Accept-Encoding'), ('server', 'gws'), ('connection', 'close'), ('cache-control', 'private, max-age=0'), ('date', 'Thu, 29 Dec 2016 09:24:02 GMT'), ('p3p', 'CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."'), ('content-type', 'text/html; charset=Shift_JIS'), ('x-frame-options', 'SAMEORIGIN')]
>>> p.headers['content-type']
'text/html; charset=Shift_JIS'

>>> s = = urllib2.urlopen("http://www.example.com")
SyntaxError: invalid syntax
>>> s = urllib2.urlopen("http://www.example.com")
>>> s.url
'http://www.example.com'
>>> s.headers.items()
[('content-length', '1270'), ('x-ec-custom-error', '1'), ('x-cache', 'HIT'), ('expires', 'Thu, 05 Jan 2017 09:28:12 GMT'), ('vary', 'Accept-Encoding'), ('server', 'ECS (rhv/818F)'), ('last-modified', 'Fri, 09 Aug 2013 23:54:35 GMT'), ('connection', 'close'), ('etag', '"359670651+gzip+ident"'), ('cache-control', 'max-age=604800'), ('date', 'Thu, 29 Dec 2016 09:28:12 GMT'), ('content-type', 'text/html')]

parsing xml
from xml.com import minidom

json

>>> import json
>>> j = '{"one": 1, "numbers": [1,2,3.5]}'
>>> json.loads(j)
{u'numbers': [1, 2, 3.5], u'one': 1}

def total_ups():
    j = json.loads(reddit_front)
    sum(c['data']['ups'] for c in j['data']['children'])

host ip info
host ip info

IP_URL = "http://api.hostip.info/?ip="
def get_coords(ip):
    url = IP_URL + ip
    content = None
    content = urllib2.urlopen(url).read()
    except URLError:
        return
    if content:
        d = minidom.parseString(content)
        coords = d.getElementByTagName("gml:coordinates")
        if coords and coords[0].childNodes[0].nodeValue:
            lon, lat = coords[0].childNodes[0].nodeValue.split(',')
            return db.GetPt(lat, lon)

Gmap

GMAPS_URL = ""

def gmap_img(points):
    markers = '&'.join('makers=%s,%s' % (p.lat, p.lon)
        for p in points)
    return GMAPS_URL + markers

print gmaps_img([Point])

Hashing

What is a hash?
H(x) -> y

ex. crc32 – checksums
md5 – fast
sha1 – secure
sha256 -pretty good

set-cookie:visit = 5, [hash]

making a hash

import hashlib

def hash_str(s):
    return hashlib.md5(s).hexdigest()

def make_secure_val(s):
    return "%s, %s" % (s, hash_string(s))

checking correct hash

def check_secure_val(h):
    val = h.split('.')[0]
    if h == make_secure_val(val):
        return val

    def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        visits = 0
        visits = self.request.cookies.get('visits')
        if visit_cookie_val:
            cookie_val = check_secure_val(visit_cookie_str)
            if cookie_val:
                visits = ini(cookie_val)
        
        visits += 1

HMAC is hash-based message authentication code
hmac(secret, key, h)-> [HASH]

$ hmac.new(“secret”, “hoge”).hexdigest()

import hashlib
import hmac

SECRET = 'imsosecret'
def hash_str(s):
    return hmac.new(SECRET, s).hexdigest()

def make_secure_val(s):
    return "%s|%s" % (s, hash_str(s)) 

def check_secure_val(h):
    val = h.split('|')[0]
    if h == make_secure_val(val):
        return val

database should change password hashing
random function in python

def make_salt():
    return ''.(random.choice(string.letters) for x in xrange(5))

def make_pw_hash(name, pw):
    salt = make_salt()
    h = hashlib.sha256(name + pw * salt).hexdigest()
    return '%s,%s' % (h, salt)

Python Set-Cookie

Set-Cookie: name=steve; Domain=www.rddit.com; Path=/

domain is restricted wwww.

Third party set cookie such as google analytics
ad network also set cookie

Set-Cookie: user=123; Expire= Ture, 1 Jan
"session" cookie = no Expire

session cookie delete when close the browser.

def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        visits = self.request.cookies.get('visits', 0)
        if visits.isdigit();
            visits = int(visits) + 1
        else:
            visits = 0
        self.response.headers.add_header('Set-Cookie', 'visits=%s' % visits)
        if visits > 100:
            self.write("you are the best ever")
        else:
            self.write("you've been here %s times!" % visits)

write document.cookie in console.
we can rewrite cookie.

document.cookie

"wordpress_test_cookie=WP+Cookie+check; wp-settings-1=editor%3Dhtml%26libraryContent%3Dbrowse%26imgsize%3Dmedium; wp-settings-time-1=1482994300; _ga=GA1.2.511761152.1479929467"

document.cookie="wp-settings-tims-1=1482994301"

"wp-settings-tims-1=1482994301"

db connection

db. = sqlite3.connect(':memory:')
db.execute('create table links ' +
            '(id integer, submitter_id integer, submitted_time integer, ' +
            'votes integer, title text, url text)')
for l in links:
    db.execute('insert into links values (?, ?, ?, ?, ?, ?)', l)

def query():
    c = db.execute("select * from links")

def query():
    cursor = db.execute("select * from links")
    for link_tuple in cursor:
        link = Link(*link_tuple)

print query()

def query():
    cursor = db.execute("select * from links where submitter_id = 62443 and votes > 1000")
    link = Link(*c.fetchone())
    return link.id

order by

def query():
    cursor = db.execute("select * from links where submitter_id = 62443 order by submitter_time asc")
    for link_tuple in c:
        link = Link(*link_tuple)
        results.append(link.id)
    return results

join

def link_by_id(link_id):
    for l in links:
        if l.id == link_id:
            return l

scaling database is to 1.replicate, 2.shared.

Querying

from collections import namedtuple

# make a basic Link class
Link = namedtuple('Link', ['id', 'submitter_id', 'submitted_time', 'votes',
                           'title', 'url'])

# list of Links to work with
links = [
    Link(0, 60398, 1334014208.0, 109,
         "C overtakes Java as the No. 1 programming language in the TIOBE index.",
         "http://pixelstech.net/article/index.php?id=1333969280"),
    Link(1, 60254, 1333962645.0, 891,
         "This explains why technical books are all ridiculously thick and overpriced",
         "http://prog21.dadgum.com/65.html"),
    Link(23, 62945, 1333894106.0, 351,
         "Learn Haskell Fast and Hard",
         "http://yannesposito.com/Scratch/en/blog/Haskell-the-Hard-Way/"),
    Link(2, 6084, 1333996166.0, 81,
         "Announcing Yesod 1.0- a robust, developer friendly, high performance web framework for Haskell",
         "http://www.yesodweb.com/blog/2012/04/announcing-yesod-1-0"),
    Link(3, 30305, 1333968061.0, 270,
         "TIL about the Lisp Curse",
         "http://www.winestockwebdesign.com/Essays/Lisp_Curse.html"),
    Link(4, 59008, 1334016506.0, 19,
         "The Downfall of Imperative Programming. Functional Programming and the Multicore Revolution",
         "http://fpcomplete.com/the-downfall-of-imperative-programming/"),
    Link(5, 8712, 1333993676.0, 26,
         "Open Source - Twitter Stock Market Game - ",
         "http://www.twitstreet.com/"),
    Link(6, 48626, 1333975127.0, 63,
         "First look: Qt 5 makes JavaScript a first-class citizen for app development",
         "http://arstechnica.com/business/news/2012/04/an-in-depth-look-at-qt-5-making-javascript-a-first-class-citizen-for-native-cross-platform-developme.ars"),
    Link(7, 30172, 1334017294.0, 5,
         "Benchmark of Dictionary Structures", "http://lh3lh3.users.sourceforge.net/udb.shtml"),
    Link(8, 678, 1334014446.0, 7,
         "If It's Not on Prod, It Doesn't Count: The Value of Frequent Releases",
         "http://bits.shutterstock.com/?p=165"),
    Link(9, 29168, 1334006443.0, 18,
         "Language proposal: dave",
         "http://davelang.github.com/"),
    Link(17, 48626, 1334020271.0, 1,
         "LispNYC and EmacsNYC meetup Tuesday Night: Large Scale Development with Elisp ",
         "http://www.meetup.com/LispNYC/events/47373722/"),
    Link(101, 62443, 1334018620.0, 4,
         "research!rsc: Zip Files All The Way Down",
         "http://research.swtch.com/zip"),
    Link(12, 10262, 1334018169.0, 5,
         "The Tyranny of the Diff",
         "http://michaelfeathers.typepad.com/michael_feathers_blog/2012/04/the-tyranny-of-the-diff.html"),
    Link(13, 20831, 1333996529.0, 14,
         "Understanding NIO.2 File Channels in Java 7",
         "http://java.dzone.com/articles/understanding-nio2-file"),
    Link(15, 62443, 1333900877.0, 1244,
         "Why vector icons don't work",
         "http://www.pushing-pixels.org/2011/11/04/about-those-vector-icons.html"),
    Link(14, 30650, 1334013659.0, 3,
         "Python - Getting Data Into Graphite - Code Examples",
         "http://coreygoldberg.blogspot.com/2012/04/python-getting-data-into-graphite-code.html"),
    Link(16, 15330, 1333985877.0, 9,
         "Mozilla: The Web as the Platform and The Kilimanjaro Event",
         "https://groups.google.com/forum/?fromgroups#!topic/mozilla.dev.planning/Y9v46wFeejA"),
    Link(18, 62443, 1333939389.0, 104,
         "github is making me feel stupid(er)",
         "http://www.serpentine.com/blog/2012/04/08/github-is-making-me-feel-stupider/"),
    Link(19, 6937, 1333949857.0, 39,
         "BitC Retrospective: The Issues with Type Classes",
         "http://www.bitc-lang.org/pipermail/bitc-dev/2012-April/003315.html"),
    Link(20, 51067, 1333974585.0, 14,
         "Object Oriented C: Class-like Structures",
         "http://cecilsunkure.blogspot.com/2012/04/object-oriented-c-class-like-structures.html"),
    Link(10, 23944, 1333943632.0, 188,
         "The LOVE game framework version 0.8.0 has been released - with GLSL shader support!",
         "https://love2d.org/forums/viewtopic.php?f=3&t=8750"),
    Link(22, 39191, 1334005674.0, 11,
         "An open letter to language designers: Please kill your sacred cows. (megarant)",
         "http://joshondesign.com/2012/03/09/open-letter-language-designers"),
    Link(21, 3777, 1333996565.0, 2,
         "Developers guide to Garage48 hackatron",
         "http://martingryner.com/developers-guide-to-garage48-hackatron/"),
    Link(24, 48626, 1333934004.0, 17,
         "An R programmer looks at Julia",
         "http://www.r-bloggers.com/an-r-programmer-looks-at-julia/")]


# links is a list of Link objects. Links have a handful of properties. For
# example, a Link's number of votes can be accessed by link.votes if "link" is a
# Link.

# make the function query() return the number of votes for the link whose ID is
# 15

def query():
    submissions = []
    for l in links:
        if submitter_id = 62443:
            submissions.append(l)
    submissions.sort(key = lambda x: x.submitted_time)
    return submissions
print query()

template

import os
import webapp2

form_html = """
<form>
<h2>Add a Food</h2>
<input type="text" name="food">
%s 
<button>Add</button>
</form>
"""
hidden_html = """
<input type="hidden" name="food" value="%s">
"""

shopping_list_html = """
<br>
<br>
<h2>Shopping List</h2>
<ul>
%s
</ul>
"""

class Handler(webapp2.RequestHandler):
    def write(self, *a, **kw):
        self.response.out.write(*a, **kw)

class MainPage(Handler):
    def get(self):
        output = form_html
        hidden_html = ""

        items = self.request.get_all("food")
        if items:
        output_items = ""
        for item in items:
            output_hidden += hidden_html % item
            output_items += item_html % item

        output_shopping = shopping_list_html % output_items
        output += output_shopping

        output = output % output_hidden
            
        self.write(output)

app = webapp2.WSGIApplication([('/', MainPage),
                    ],
                    debug=True)

application:template-lesson
version: 1
runtime: python27
api_version: 1
threadsafe: True

handlers:
  - url: /.*
    script: templates.app

python template

<!DOCTYPE html>

<html>
  <head>
    <title>templates!</title>
  </head>

  <body style="margin: 0">
    <h1 style="background-color: #ddd: color: #888; margin: 0, height: 50px">
      Templates
    </h1>
    {% block content %}
    {% endblock %}
  </body>
</html>

{% extends "base.html" %}

{% block content %}
<form>
      <h2>Add a Food</h2>
      <input type="text" name="food">
      {% if items %}
        {% for item in items %}
          <input type="hidden" name="food" value="{{item}}">
        {% endfor %}
      <button>Add</button>

      {% if items %}
      <br>
      <br>

      <h2>Shopping List</h2>
      <ul>
        {% for item in items %}
        <li>{{ item | escape }}</li>
      </ul>
      {% endif %}
</form>
{% endblock %}

rot13

def _rot13(c):
if ‘A’ <= c and c <= 'Z': return chr((ord(c) - ord('A') + 13) % 26 + ord('A')) if 'a' <= c and c <= 'Z': return chr((ord(c) - ord('a') + 13) % 26 + ord('a')) return c 
def render_str(template, **params):
	t = jinja_env.get_template(template)
	return t.render(params)

class BaseHandler(webapp2.RequestHandler):
	def render(self, template, **kw):
		self.response.out.write(render_str(template, **kw))

	def write(self, *a, **kw):
		self.response.out.write(*a, **kw)

class Rot13(BaseHandler):
	def get(self):
		self.render('rot13-form.html')

	def post(self)
		rot13 = ''
		text = self.request.get('text')
		if text:
			rot13 = text.encode('rot13')

		self.render('rot13-form.html', text = rot13)

USER_RE = re.compile(r"^[a-zA-Z0-9_-]{3,20}$")
def valid_username(username):
	return username and USER_RE.match(username)

PASS_RE = re.compile(r"^.{3,20}$")
def valid_password(password):