Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

370
LINES

< > TinyBrain | #1002576 - Eleu Web Serving (Include)

JavaX fragment (include)

static Set<S> blockedIPs = synchroHashSet();
//static int blockedIPDelay = -1; // forever
static int blockedIPDelay = 10; // seconds

//static int spamInterval = 60000;
//static double spamPerSecondThreshold = 2.0; // per IP + URI

// for sub-bots to call through reflection
please include function serveRedirect.
please include function serve404.
please include function serveFile.
please include function serveFile_maxCache.
please include function serveByteArray.
please include function serveByteArray_maxCache.

static volatile int hitCount;

// These are speaking URLs like "/images/"
static SS botNames = litmap(
  serve := "#1014036/raw",
  img := "#1014038/raw",
  mech := #1013927,
  images := #1004590,
  eleu := "#1002213/raw",
  "eleuraw" := "#1002076/raw",
  "don-enrico" := "#1006604/raw",
  wiki := "#1007510/raw",
  jobs := "#1007647/raw",
  chess := "#1012832/raw");

static S anonymousDialogID = "web-anon";

static S webLogProgramID = "#1002576";
static S webLog = "webLog";

static int recentWebLogMax = 1000;
static new L<Map> recentWebLog; // TODO: init from log at startup

static S homePageBotID = "#1002771";

static void webInit() {
  readLocally("hitCount");
  readLocally("homePageBotID");
  //readLocally("blockedIPs");
  set NanoHTTPD_debug;
}

static new Object webLock;

// for detecting bad clients
// ip + uri -> count
//static ExpiringHashMap<Pair<S>, Int> ipsAndUris = new ExpiringHashMap(spamInterval).dontRenewOnOverwrite();

static NanoHTTPD.Response serve(S uri, NanoHTTPD.Method method,
  Map<S,S> header, Map<S,S> params, Map<S,S> files) {
  
  S clientIP = getSession().getHeaders().get("remote-addr");
  
  Pair<Int, Bool> pair = simpleSpamClientDetect2(clientIP, uri);
  //Pair<S> pair = pair(clientIP, uri);
  //int ipCount = toInt(ipsAndUris.get(pair));
  //ipsAndUris.put(pair, ipCount+1);

  bool bad = eq(uri, "/simulate-bad-client");
  //int spamMax = iround(spamPerSecondThreshold*spamInterval/1000);
  int ipCount = pair.a;
  
  if (!bad && pair.b) {
    print("Blocking IP " + clientIP + ", count reached: " + ipCount);
    addBlockedIP(clientIP);
    bad = true;
  }
  
  if (blockedIPs.contains(clientIP)) {
    bad = true;
    if (blockedIPDelay < 0)
      print(formatDateAndTime() + " Blocked IP!! " + clientIP + " " + l(nanoHttpd_badClients()));
    else {
      print(formatDateAndTime() + " Blocked IP!! " + clientIP + (blockedIPDelay != 0 ? " Delaying " + blockedIPDelay + "s" : ""));
      sleepSeconds(blockedIPDelay);
    }
    //ret serveHTML("No");
  }
  
  if (bad) {
    NanoHTTPD.IHTTPSession httpSession = NanoHTTPD.currentSession!;
    if (httpSession == null || blockedIPDelay >= 0) {
      if (httpSession == null) print("No HTML session?");
      sleepSeconds(blockedIPDelay);
      ret serveHTML("go away");
    }
    httpSession.badClient(true);
    null;
  }
  
  print(gmtWithSeconds() + ": Serving HTTP " + quote(uri) + " to " + clientIP + " (anti-spam count: " + ipCount + "/" + simpleSpamClientDetect2_spamMax + ")");
  
  S host = header.get("host");
  if (swic(host, "smartbot.")) {
    loadBinaryPage_extraHeaders.set(litmap("User-Agent" := header.get("user-agent")));
    byte[] page = loadBinaryPage(smartBotURL() + /*urlencodeExceptSlash*/(uri) + htmlQuery(mapWithoutKey(params, "NanoHttpd.QUERY_STRING")));
    Map<S, L<S>> headers = loadBinaryPage_responseHeaders!;
    S contentType = or2(first(lookupPossiblyIgnoringCase(headers, "content-type")), "text/html; charset=utf8");
    ret serveByteArray(page, contentType);
  }
  
  long hitID;
  
  synchronized(webLock) {
    ++hitCount;
    hitID = hitCount;
    saveLocally("hitCount");
  }
  
  pcall { webLog(litmap("time", now(), "request", hitID, "uri", uri, "method", str(method), "header", header, "params", params, "files", files)); }
  
  try {
    NanoHTTPD.Response response = serveNoLog(uri, method, header, params, files);
    
    // log that we responded
    pcall { webLog(litmap("time", now(), "response", hitID)); }
    
    ret response;
  } catch (Throwable e) {
    // log that there was an error
    pcall { webLog(litmap("time", now(), "response-error", hitID, getStackTrace(e))); }
    throw asRuntimeException(e);
  }
}
  
static NanoHTTPD.Response serveNoLog(S uri, NanoHTTPD.Method method,
  Map<S,S> header, Map<S,S> params, Map<S,S> files) {
  
  actualURI.set(uri);
  if (nempty(files))
    print("Files: " + l(files));
  uploadFiles.set(files);
  uri = dropPrefixMandatory("/", uri);
  
  if (!loaded) {
    int numLoaded = l(bots), total = l(botIDs);
    
    // query dispatcher for number of loaded sub-bots
    
    S lbi = loadingBotID;
    if (sameSnippetID(lbi, "#1002317")) {
      int[] x = (int[]) callOpt(loadingBot, "numLoaded");
      if (x != null) {
        numLoaded += x[0];
        total += x[1];
      }
      
      lbi = or((S) getOpt(loadingBot, "loadingBotID"), lbi);
    }
    
    S s = "LOADING, PLEASE TRY AGAIN. " + numLoaded + "/" + total + " bots loaded ";
    if (lbi != null) 
      s +=  " (loading bot " + formatSnippetID(lbi) + " - " + getSnippetTitle_overBot(lbi) + ")";
    
    ret serveHTML(s);
  }
  
  // count and set cookie
  O visitorsBot = getBot("#1002157");
  if (visitorsBot != null)
    callHtmlMethod(visitorsBot, "/");

  setDialogIDForWeb();
  
  if (eq(uri, "") || eq(uri, "raw"))
    ret serveHomePage("/", params);
  if (eq(uri, "favicon.ico"))
    ret serveFile(loadLibrary(#1013028), "image/x-icon");

  int i = uri.indexOf('/');
  if (i < 0) { uri += "/"; i = l(uri)-1; }
  S firstPart = uri.substring(0, i);
  
  print("uri: " + uri + ", i: " + i + ", firstPart: " + firstPart);
  if (botNames.containsKey(firstPart)) {
    uri = dropPrefix("#", botNames.get(firstPart))
      + addPrefix("/", substring(uri, i));
    i = uri.indexOf('/');
    firstPart = i >= 0 ? uri.substring(0, i) : uri;
    print("now firstPart: " + firstPart + ", uri: " + uri + ", i: " + i);
  }
  
  S rest = i >= 0 ? substr(uri, i) : "/"; // rest always starts with "/"
  
  if (possibleGlobalID(toLower(firstPart))) {
    ret serveBot(#1007510, "/raw/" + uri, params);
    /*AIConcept c = aiConceptsMap_cached().get(toLower(firstPart));
    if (c != null)
      ret serveHTML("Concept found: " + c.globalID + " - " + c.name);
    else
      ret serveHTML("Concept not found: " + toLower(firstPart));*/
  }

  if (!isInteger(firstPart))
    ret serveHomePage("/" + uri, params);
    
  ret serveBot(firstPart, rest, params);
}

static NanoHTTPD.Response serveBot(S botID, S subUri, Map<S, S> params) {
  Class bot = getBot(botID);
  boolean raw = subUri.equals("/raw") || subUri.startsWith("/raw/");
  if (raw) {
    subUri = substr(subUri, "/raw".length());
    if (subUri.length() == 0) subUri = "/";
  }
  if (bot != null) {
    //print("Bot " + botID + " methods: " + structure(allMethodNames(bot)));
    O botOut = callExtendedHtmlMethod(bot, subUri, params);
    if (botOut instanceof NanoHTTPD.Response)
      ret (NanoHTTPD.Response) botOut;
    if (eq(getClassName(botOut), NanoHTTPD.Response.class.getName()))
      fail("Wrong realm");
      
    S botHtml = str(botOut);
    S html;
    if (raw) html = unnull(botHtml);
    else {
      if (botHtml == null)
        botHtml = "Bot has no HTML output.";
      S name = getSnippetTitle(botID);
      S title = htmlencode(name) + " [" + formatSnippetID(botID) + "]";
      html = "<html><head><title>" + title + "</title></head>" +
        "<body><h3>Bot <a href=\"http://tinybrain.de/" + parseSnippetID(botID) + "\">" + formatSnippetID(botID) + "</a> - " + htmlencode(name) + "</h3>\n" + botHtml +
        "\n</body></html>";
    }
    ret serveHTMLNoCache(html);
  }
  
  ret serve404();
}

static NanoHTTPD.Response serveHomePage(S uri, Map<S, S> params) {
  if (nempty(homePageBotID)) {
    O bot = getBot(homePageBotID);
    if (bot != null) pcall {
      ret serveHTMLNoCache(callHtmlMethod(bot, uri, params));
    }
  }

  new StringBuilder buf;
  buf.append("<html>");
  buf.append("<head><title>TinyBrain Chat Bots</title></head>");
  buf.append("<body>");
  buf.append("<h3><a href=" + htmlQuote("http://tinybrain.de") + ">TinyBrain</a> Bots</h3>");
  buf.append("<ul>");
  
  L<S> botsToList = litlist(getProgramID());
  O dispatcher = getDispatcher();
  if (dispatcher != null)
    botsToList.addAll((L) call(dispatcher, "getSubBotIDs"));
  botsToList.addAll(activeBots);
  
  for (S botID : botsToList) {
    botID = formatSnippetID(botID);
    S parsedID = "" + parseSnippetID(botID);
    S title = "?";
    pcall { title = getSnippetTitle_overBot(botID); }
    S name = botID + " - " + htmlencode(title);
    buf.append("<li><a href=" + htmlQuote(parsedID) + ">" + name + "</a> (");
    buf.append("<a href=" + htmlQuote("http://tinybrain.de/" + parsedID) + ">source</a>)</li>");
  }
  buf.append("</ul>");
  buf.append("<p>Hit count: " + hitCount + "</p>");
  
  buf.append("<p>Last interactions:</p>");
  
  int maxInteractionsToPrint = 10;
  
  for (int i = l(history)-1; i >= 0 && i >= l(history)-maxInteractionsToPrint; i--) {
    Map m = history.get(i);
    buf.append("<p>" + htmlencode(m.get("question")) + "<br>&nbsp; &nbsp; " + htmlencode(m.get("answer")) + "</p>");
  }
  
  buf.append("</body>");
  buf.append("</html>");
  ret serveHTML(buf);
}

// for sub-bots to call
static NanoHTTPD.IHTTPSession getSession() {
  ret NanoHTTPD.currentSession.get();
}

// for sub-bots to call
static O getCookies() {
  NanoHTTPD.IHTTPSession session = NanoHTTPD.currentSession.get();
  NanoHTTPD.CookieHandler cookies = session.getCookies();
  ret cookies;
}

static void setDialogIDForWeb() {
  O session = getSession();
  O cookieHandler = getCookies();
  S cookie = cast call(cookieHandler, "read", "cookie");
  //print("Web answer cookie: " + cookie);
  //print("Web answer question: " + quote(s));
  S dialogID = nempty(cookie) ? "web-" + hashCookie(cookie) : anonymousDialogID;
  main.dialogID.set(dialogID);
}

static S webAnswer(S s) {
  ret webAnswer(s, true);
}

static S webAnswer(S s, boolean log) {
  setDialogIDForWeb();
  
  originalLine.set(s);
  
  // drop the attn prefix ("!")
  s = dropPrefix("!", s).trim();
  
  attn.set(true);
  dedicated.set(true);
  // user name is empty for now
        
  S a = null;
  pcall {
    a = answer(s, false); // generalRelease = false, all bots
    print("Web answer: " + quote(a));
    if (empty(a))
      a = "Hello :)";
  }
  if (empty(a)) a = "(no response)";
  
  if (log) {
    S user = getDialogID();
    Map logEntry = litmap("question", s, "user", user, "answer", a);
    historyAdd(logEntry);
  }
  
  ret a;
}

static S hashCookie(S cookie) {
  ret md5(cookie).substring(0, 10);
}

static void webLog(Map data) {
  synchronized(webLock) {
    logStructure(getProgramFile(webLogProgramID, webLog), data);
    if (recentWebLogMax > 0) {
      if (l(recentWebLog) >= recentWebLogMax)
        recentWebLog.remove(0); // yeah... I know, O(1000)
      recentWebLog.add(data);
    }
  }
}

static L<Map> getRecentWebLog() {
  synchronized(webLock) {
    ret cloneList(recentWebLog);
  }
}

static synchronized void addBlockedIP(S ip) {
  blockedIPs.add(ip);
  //save("blockedIPs");
}

static synchronized void removeBlockedIP(S ip) {
  blockedIPs.remove(ip);
  //save("blockedIPs");
}

download  show line numbers  debug dex   

Travelled to 5 computer(s): cfunsshuasjs, gwrvuhgaqvyk, mqqgnosmbjvj, onxytkatvevr, tvejysmllsmz

No comments. add comment

Snippet ID: #1002576
Snippet name: Eleu Web Serving (Include)
Eternal ID of this version: #1002576/75
Text MD5: 27c4dfc84db2189224fd2ce77e965502
Author: stefan
Category: eleu
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2018-04-16 14:09:28
Source code size: 11594 bytes / 370 lines
Pitched / IR pitched: No / No
Views / Downloads: 529 / 628
Version history: 74 change(s)
Referenced in: [show]