Posthog Data Pipelines

Data piplines are an advanced Posthog feature that allows you to directly modify or filter out incoming data.

Here's an example of one I experiment with, which demonstrates some advanced techniques.

AI Traffic Example

// Hog transformation: tag events whose IPv4 is in ChatGPT CIDRs.
// Always returns the event (pass-through if no match).

fun ipToInt(ip) {
    if (ip = null) { return null }
    // Skip IPv6
    if (match(ip, ':')) { return null }

    let parts := splitByString('.', ip)
    if (length(parts) != 4) { return null }

    let a := toInt(parts[1])
    let b := toInt(parts[2])
    let c := toInt(parts[3])
    let d := toInt(parts[4])

    if (a < 0 or a > 255 or b < 0 or b > 255 or c < 0 or c > 255 or d < 0 or d > 255) { return null }
    return a*16777216 + b*65536 + c*256 + d
}

fun pow2(n) {
    let r := 1
    let i := 0
    while (i < n) {
        r := r * 2
        i := i + 1
    }
    return r
}

fun cidrToRange(cidr) {
    let p := splitByString('/', cidr)
    if (length(p) != 2) { return null }
    let base := p[1]
    let bits := toInt(p[2])
    if (bits < 0 or bits > 32) { return null }

    let baseInt := ipToInt(base)
    if (baseInt = null) { return null }

    let hostBits := 32 - bits
    let size := pow2(hostBits)
    let start := toInt(baseInt / size) * size
    let end := start + size - 1
    return (start, end)
}

// ChatGPT-User CIDR source (2025-09-11T23:02:11.854419)
let CHATGPT_USER_CIDRS := [
'23.98.142.176/28','23.102.140.144/28','13.65.138.112/28','23.98.179.16/28','13.65.138.96/28',
'172.183.222.128/28','20.102.212.144/28','40.116.73.208/28','172.183.143.224/28','52.190.190.16/28',
'13.83.237.176/28','51.8.155.64/28','74.249.86.176/28','51.8.155.48/28','20.55.229.144/28',
'135.237.131.208/28','135.237.133.48/28','51.8.155.112/28','135.237.133.112/28','52.159.249.96/28',
'52.190.137.16/28','52.255.111.112/28','40.84.181.32/28','172.178.141.112/28','52.190.142.64/28',
'172.178.140.144/28','52.190.137.144/28','172.178.141.128/28','57.154.187.32/28','4.196.118.112/28',
'20.193.50.32/28','20.215.188.192/28','20.215.214.16/28','4.197.22.112/28','4.197.115.112/28',
'172.213.21.16/28','172.213.11.144/28','172.213.12.112/28','172.213.21.144/28','20.90.7.144/28',
'57.154.175.0/28','57.154.174.112/28','52.236.94.144/28','137.135.191.176/28','23.98.186.192/28',
'23.98.186.96/28','23.98.186.176/28','23.98.186.64/28','68.221.67.192/28','68.221.67.160/28',
'13.83.167.128/28','20.228.106.176/28','52.159.227.32/28','68.220.57.64/28','172.213.21.112/28',
'68.221.67.224/28','68.221.75.16/28','20.97.189.96/28','52.252.113.240/28','52.230.163.32/28',
'172.212.159.64/28','52.255.111.80/28','52.255.111.0/28','4.151.241.240/28','52.255.111.32/28',
'52.255.111.48/28','52.255.111.16/28','52.230.164.176/28','52.176.139.176/28','52.173.234.16/28',
'4.151.71.176/28','4.151.119.48/28','52.255.109.112/28','52.255.109.80/28','20.161.75.208/28',
'68.154.28.96/28','52.255.109.128/28','52.255.109.96/28','52.255.109.144/28','52.173.234.80/28',
'132.196.82.48/28','20.249.63.208/28','20.63.221.64/28','13.76.116.80/28','20.235.87.224/28',
'4.205.128.176/28','52.225.75.208/28','52.190.139.48/28','68.221.67.240/28','40.75.14.224/28',
'135.119.134.192/28','51.8.155.80/28','135.119.134.128/28','52.173.219.112/28','52.242.132.224/28',
'52.173.219.96/28','52.242.132.240/28','74.7.36.64/28','74.7.36.96/28','74.7.35.48/28',
'74.7.35.112/28','74.7.36.80/28','52.156.77.144/28','52.148.129.32/28','20.117.22.224/28',
'20.235.75.208/28','172.204.16.64/28','4.196.198.80/28','20.194.157.176/28','23.102.141.32/28',
'52.173.235.80/28','52.173.123.0/28','40.84.221.208/28','104.210.139.224/28','20.0.53.96/28',
'52.154.22.48/28','52.242.245.208/28','191.235.66.16/28','191.233.196.112/28','191.233.194.32/28',
'23.97.109.224/28','138.91.46.96/28','13.76.32.208/28','52.187.246.128/28','13.70.107.160/28',
'138.91.30.48/28','20.210.154.128/28','20.194.1.0/28','20.194.0.208/28','20.77.178.240/28',
'4.234.83.96/28','40.84.221.224/28','104.210.139.192/28','191.239.245.16/28','191.234.167.128/28',
'191.235.99.80/28','191.235.98.144/28','68.218.30.112/28','4.197.19.176/28','20.27.94.128/28',
'20.210.174.208/28','20.204.24.240/28','20.198.67.96/28'
]

// GPTBot (2025-05-22T11:51:00)
let GPTBOT_CIDRS := [
'52.230.152.0/24','20.171.206.0/24','20.171.207.0/24','4.227.36.0/25',
'20.125.66.80/28','172.182.204.0/24','172.182.214.0/24','172.182.215.0/24'
]

// SearchBot (2025-06-11T12:00:00)
let SEARCHBOT_CIDRS := [
'20.42.10.176/28','172.203.190.128/28','104.210.140.128/28','51.8.102.0/24','135.234.64.0/24',
'172.182.195.48/28','20.25.151.224/28','20.171.53.224/28','20.169.6.224/28','172.182.193.80/28',
'172.182.193.224/28','172.182.194.32/28','172.182.194.144/28','172.182.213.192/28','172.182.209.208/28',
'172.182.224.0/28','172.182.211.192/28','20.169.7.48/28','20.168.18.32/28','20.171.123.64/28','20.14.99.96/28'
]

// Precompute numeric ranges
let CHATGPT_USER_RANGES := []
for (let i := 1; i <= length(CHATGPT_USER_CIDRS); i := i + 1) {
    let r := cidrToRange(CHATGPT_USER_CIDRS[i])
    if (r != null) {
        CHATGPT_USER_RANGES := arrayPushBack(CHATGPT_USER_RANGES, r)
    }
}

let GPTBOT_RANGES := []
for (let i := 1; i <= length(GPTBOT_CIDRS); i := i + 1) {
    let r := cidrToRange(GPTBOT_CIDRS[i])
    if (r != null) {
        GPTBOT_RANGES := arrayPushBack(GPTBOT_RANGES, r)
    }
}

let SEARCHBOT_RANGES := []
for (let i := 1; i <= length(SEARCHBOT_CIDRS); i := i + 1) {
    let r := cidrToRange(SEARCHBOT_CIDRS[i])
    if (r != null) {
        SEARCHBOT_RANGES := arrayPushBack(SEARCHBOT_RANGES, r)
    }
}

// Copy event to make it mutable
let returnEvent := event
returnEvent.properties := returnEvent.properties ?? {}

// Prefer upstream cryptographic proof if you set it at your edge
if (returnEvent.properties['chatgpt_signature_verified'] = true) {
    returnEvent.properties['ai_traffic'] := true
    returnEvent.properties['ai_traffic_platform'] := 'chatgpt'
    returnEvent.properties['ai_traffic_type'] := 'chatgpt_user'
    returnEvent.properties['ai_traffic_identifier'] := 'signature'
    return returnEvent
}

let ip := returnEvent.properties['$ip'] ?? returnEvent.properties['ip'] ?? returnEvent['$ip']
let ipInt := ipToInt(ip)

if (ipInt != null) {
    // Check GPTBot first
    for (let i := 1; i <= length(GPTBOT_RANGES); i := i + 1) {
        if (ipInt >= GPTBOT_RANGES[i].1 and ipInt <= GPTBOT_RANGES[i].2) {
            returnEvent.properties['ai_traffic'] := true
            returnEvent.properties['ai_traffic_platform'] := 'chatgpt'
            returnEvent.properties['ai_traffic_type'] := 'gptbot'
            returnEvent.properties['ai_traffic_identifier'] := 'ip'
            return returnEvent
        }
    }
    
    // Check SearchBot
    for (let i := 1; i <= length(SEARCHBOT_RANGES); i := i + 1) {
        if (ipInt >= SEARCHBOT_RANGES[i].1 and ipInt <= SEARCHBOT_RANGES[i].2) {
            returnEvent.properties['ai_traffic'] := true
            returnEvent.properties['ai_traffic_platform'] := 'chatgpt'
            returnEvent.properties['ai_traffic_type'] := 'searchbot'
            returnEvent.properties['ai_traffic_identifier'] := 'ip'
            return returnEvent
        }
    }
    
    // Check ChatGPT User
    for (let i := 1; i <= length(CHATGPT_USER_RANGES); i := i + 1) {
        if (ipInt >= CHATGPT_USER_RANGES[i].1 and ipInt <= CHATGPT_USER_RANGES[i].2) {
            returnEvent.properties['ai_traffic'] := true
            returnEvent.properties['ai_traffic_platform'] := 'chatgpt'
            returnEvent.properties['ai_traffic_type'] := 'chatgpt_user' 
            returnEvent.properties['ai_traffic_identifier'] := 'ip'
            return returnEvent
        }
    }
}

// Set ai_traffic = false if no match
returnEvent.properties['ai_traffic'] := false

// Always pass through
return returnEvent