#!/bin/csh -f
# Script to filter stdin to stdout, stripping out lines that look like 
# log httpd server log lines from bots.

# Without the --line-buffered option, grep buffers its output.
# So when used in a pipe, it may never actually emit anything out of 
# the end of the pipe until the data stream ends.  That's fine for 
# batch processes, and it's more efficient.  But, it's a problem 
# when feeding in live data like tail -F and expecting to see output 
# immediately.  Especially when used with other pipe elements that 
# buffer their output (grep, sed, others?).  Until all of the buffers
# of the entire pipe fill up, you get no output.  And if you kill the
# process with Ctrl-C (since tail -F never ends), any buffered output
# is lost.
set grep = "grep --line-buffered"
$grep -v Googlebot \
| $grep -v bingbot \
| $grep -v MJ12bot \
| $grep -v openai.com \
| $grep -v gptbot \
| $grep -v amazonbot \
| $grep -v SemrushBot \
| $grep -v DotBot \
| $grep -v DomainStatsBot \
| $grep -v AhrefsBot \
| $grep -v YandexBot \
| $grep -v Bytespider \
| $grep -v AspiegelBot \
| $grep -v SeznamBot \
| $grep -v AlphaBot \
| $grep -v PetalBot \
| $grep -v MegaIndex.ru \
| $grep -v Applebot \
| $grep -v MauiBot \
| $grep -v LivelapBot \
| $grep -v PaperLiBot \
| $grep -v Adsbot \
| $grep -v Cocolyzebot \
| $grep -v BLEXBot \
| $grep -v "The Knowledge AI" \
| $grep -v ZoominfoBot \
| $grep -v megagrabber.ru \