Crabler - Web crawler for Crabs

Structures as asynchronous web crawlers.

Example

```rust extern crate crabler;

use crabler::*;

[derive(WebScraper)]

[onresponse(responsehandler)]

[onhtml("a[href]", printhandler)]

[onhtml("td.title > a.storylink[href]", followhandler)]

struct Scraper {}

impl Scraper { async fn response_handler(&self, response: Response) -> Result<()> { println!("Status {}", response.status); Ok(()) }

async fn print_handler(&self, response: Response, a: Element) -> Result<()> {
    if let Some(href) = a.attr("href") {
        println!("Found link {} on {}", href, response.url);
    }

    Ok(())
}

async fn follow_handler(&self, mut response: Response, a: Element) -> Result<()> {
    if let Some(href) = a.attr("href") {
        response.navigate(href).await?;
    }

    Ok(())
}

}

[tokio::main]

async fn main() -> Result<()> { let scraper = Scraper { };

let mut crabweb = Crabler::new(scraper);

// Queue navigation task
crabweb.navigate("https://news.ycombinator.com/").await?;

// Create bunch of http workers
for _ in 0..20 {
    crabweb.start_worker();
}

// Run main scraper loop
crabweb.run().await?;

Ok(())

} ```