/*
    *  This file is part of the Heritrix web crawler (crawler.archive.org).
    *
    *  Licensed to the Internet Archive (IA) by one or more individual 
    *  contributors. 
    *
    *  The IA licenses this file to You under the Apache License, Version 2.0
    *  (the "License"); you may not use this file except in compliance with
    *  the License.  You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   *  Unless required by applicable law or agreed to in writing, software
   *  distributed under the License is distributed on an "AS IS" BASIS,
   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   *  See the License for the specific language governing permissions and
   *  limitations under the License.
   */
  package org.archive.wayback.accesscontrol.robotstxt;
  
  import java.io.BufferedReader;
  import java.io.IOException;
  import java.io.Serializable;
  import java.util.HashMap;
  import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  import java.util.logging.Level;
  import java.util.logging.Logger;
  
  import org.apache.commons.io.IOUtils;
  import org.archive.io.ReadSource;
  
  /**
   * Utility class for parsing and representing 'robots.txt' format 
   * directives, into a list of named user-agents and map from user-agents 
   * to RobotsDirectives. 
   */
  public class Robotstxt implements Serializable {
      static final long serialVersionUID = 7025386509301303890L;
      private static final Logger logger =
          Logger.getLogger(Robotstxt.class.getName());
  
      // all user agents contained in this robots.txt
      // in order of declaration
      // TODO: consider discarding irrelevant entries
      LinkedList<String> namedUserAgents = new LinkedList<String>();
      // map user-agents to directives
      Map<String,RobotsDirectives> agentsToDirectives = 
          new HashMap<String,RobotsDirectives>();
      RobotsDirectives wildcardDirectives = null; 
      
      boolean hasErrors = false;
      
      static RobotsDirectives NO_DIRECTIVES = new RobotsDirectives();
      /** empty, reusable instance for all sites providing no rules */
      public static Robotstxt NO_ROBOTS = new Robotstxt();
      
      public Robotstxt() {
      }
  
      public Robotstxt(BufferedReader reader) throws IOException {
          initializeFromReader(reader);
      }
  
      public Robotstxt(ReadSource customRobots) {
          BufferedReader reader = new BufferedReader(customRobots.obtainReader());
          try {
              initializeFromReader(reader);
          } catch (IOException e) {
              logger.log(Level.SEVERE,
                      "robots ReadSource problem: potential for inadvertent overcrawling",
                      e);
          } finally {
              IOUtils.closeQuietly(reader); 
          }
      }
  
      protected void initializeFromReader(BufferedReader reader) throws IOException {
          String read;
          // current is the disallowed paths for the preceding User-Agent(s)
          RobotsDirectives current = null;
          // whether a non-'User-Agent' directive has been encountered
          boolean hasDirectivesYet = false; 
          while (reader != null) {
              do {
                  read = reader.readLine();
                  // Skip comments & blanks
              } while ((read != null) && ((read = read.trim()).startsWith("#") ||
                  read.length() == 0));
              if (read == null) {
                  reader.close();
                  reader = null;
              } else {
                  // remove any html markup
                  read = read.replaceAll("<[^>]+>","");
                  int commentIndex = read.indexOf("#");
                  if (commentIndex > -1) {
                      // Strip trailing comment
                     read = read.substring(0, commentIndex);
                 }
                 read = read.trim();
                 if (read.matches("(?i)^User-agent:.*")) {
                     String ua = read.substring(11).trim().toLowerCase();
                     if (current == null || hasDirectivesYet ) {
                         // only create new rules-list if necessary
                         // otherwise share with previous user-agent
                         current = new RobotsDirectives();
                         hasDirectivesYet = false; 
                     }
                     if (ua.equals("*")) {
                         wildcardDirectives = current;
                     } else {
                         namedUserAgents.addLast(ua);
                         agentsToDirectives.put(ua, current);
                     }
                     continue;
                 }
                 if (read.matches("(?i)Disallow:.*")) {
                     if (current == null) {
                         // buggy robots.txt
                         hasErrors = true;
                         continue;
                     }
                     String path = read.substring(9).trim();
                     // tolerate common error of ending path with '*' character
                     // (not allowed by original spec; redundant but harmless with 
                     // Google's wildcarding extensions -- which we don't yet fully
                     // support). 
                     if(path.endsWith("*")) {
                         path = path.substring(0,path.length()-1); 
                     }
                     current.addDisallow(path);
                     hasDirectivesYet = true; 
                     continue;
                 }
                 if (read.matches("(?i)Crawl-delay:.*")) {
                     if (current == null) {
                         // buggy robots.txt
                         hasErrors = true;
                         continue;
                     }
                     // consider a crawl-delay, even though we don't 
                     // yet understand it, as sufficient to end a 
                     // grouping of User-Agent lines
                     hasDirectivesYet = true;
                     String val = read.substring(12).trim();
                     val = val.split("[^\\d\\.]+")[0];
                     try {
                         current.setCrawlDelay(Float.parseFloat(val));
                     } catch (NumberFormatException nfe) {
                         // ignore
                     }
                     continue;
                 }
                 if (read.matches("(?i)Allow:.*")) {
                     if (current == null) {
                         // buggy robots.txt
                         hasErrors = true;
                         continue;
                     }
                     String path = read.substring(6).trim();
                     // tolerate common error of ending path with '*' character
                     // (not allowed by original spec; redundant but harmless with 
                     // Google's wildcarding extensions -- which we don't yet fully
                     // support). 
                     if(path.endsWith("*")) {
                         path = path.substring(0,path.length()-1); 
                     }
                     current.addAllow(path);
                     hasDirectivesYet = true;
                     continue;
                 }
                 // unknown line; do nothing for now
             }
         }
     }
 
     /**
      * Does this policy effectively allow everything? (No 
      * disallows or timing (crawl-delay) directives?)
      * @return <b>true</b> if the map containing user-agents to directives is empty, <b>false</b> otherwise.
      */
     public boolean allowsAll() {
         // TODO: refine so directives that are all empty are also 
         // recognized as allowing all
         return agentsToDirectives.isEmpty();
     }
     
     public List<String> getNamedUserAgents() {
         return namedUserAgents;
     }
 
     /**
      * Return the RobotsDirectives, if any, appropriate for the given User-Agent
      * string. If useFallbacks is true, a wildcard ('*') directives or the default
      * of NO_DIRECTIVES will be returned, as appropriate, if there is no better
      * match. If useFallbacks is false, a null will be returned if no declared
      * directives targeted the given User-Agent.
      * 
      * @param ua String User-Agent to lookup
      * @param useFallbacks if true, fall-back to wildcard directives or 
      * default allow as needed
      * @return directives to use, or null if useFallbacks is false and no 
      * non-wildcard directives match the supplied User-Agent
      */
     public RobotsDirectives getDirectivesFor(String ua, boolean useFallbacks) {
         // find matching ua
         for(String uaListed : namedUserAgents) {
             if(ua.indexOf(uaListed)>-1) {
                 return agentsToDirectives.get(uaListed);
             }
         }
         if(useFallbacks==false) {
             return null; 
         }
         if (wildcardDirectives!=null) {
             return wildcardDirectives;
         }
         // no applicable user-agents, so empty directives
         return NO_DIRECTIVES; 
     }
 
     /**
      * Return directives to use for the given User-Agent, resorting to wildcard
      * rules or the default no-directives if necessary.
      * 
      * @param userAgent String User-Agent to lookup
      * @return directives to use
      */
     public RobotsDirectives getDirectivesFor(String userAgent) {
         return getDirectivesFor(userAgent, true);
     }
 }