Switch to Summary instead of Counter; Add IsMirroredProject to clean up stats by project

nginx.go

@@ -17,13 +17,8 @@ import (
 type NginxExporter struct {
    AccessLogPath string
 
-   // Request counts
+   responses *prometheus.SummaryVec
-   responses *prometheus.CounterVec
+   error_responses *prometheus.SummaryVec
-   error_responses *prometheus.CounterVec
-
-   // Bytes count
-   bytes_sent *prometheus.CounterVec
-   error_bytes_sent *prometheus.CounterVec
 }
 
 func NewNginxExporter(accessLogPath string) (*NginxExporter, error) {
@@ -31,42 +26,22 @@ func NewNginxExporter(accessLogPath string) (*NginxExporter, error) {
 
    return &NginxExporter{
       AccessLogPath: accessLogPath,
-      responses: prometheus.NewCounterVec(
+      responses: prometheus.NewSummaryVec(
-         prometheus.CounterOpts{
+         prometheus.SummaryOpts{
             Namespace: Namespace,
             Subsystem: subsystem,
-            Name: "responses_total",
+            Name: "responses",
-            Help: "Number of HTTP responses",
+            Help: "Summary of HTTP responses",
          },
-         []string{"project", "network", "protocol"},
+         []string{"project", "network", "protocol"}),
-      ),
+      error_responses: prometheus.NewSummaryVec(
-      error_responses: prometheus.NewCounterVec(
+         prometheus.SummaryOpts{
-         prometheus.CounterOpts{
             Namespace: Namespace,
             Subsystem: subsystem,
-            Name: "error_responses_total",
+            Name: "error_responses",
-            Help: "Number of HTTP error responses (HTTP response code not between 100 - 399)",
+            Help: "Summary of error HTTP responses",
          },
-         []string{"project", "network", "protocol"},
+         []string{"project", "network", "protocol"}),
-      ),
-      bytes_sent: prometheus.NewCounterVec(
-         prometheus.CounterOpts{
-            Namespace: Namespace,
-            Subsystem: subsystem,
-            Name: "responses_sent_bytes",
-            Help: "Number of bytes sent in HTTP responses",
-         },
-         []string{"project", "network", "protocol"},
-      ),
-      error_bytes_sent: prometheus.NewCounterVec(
-         prometheus.CounterOpts{
-            Namespace: Namespace,
-            Subsystem: subsystem,
-            Name: "error_responses_sent_bytes",
-            Help: "Number of bytes sent in error HTTP responses (HTTP response code not between 100 - 399)",
-         },
-         []string{"project", "network", "protocol"},
-      ),
    }, nil
 }
 
@@ -74,19 +49,15 @@ func NewNginxExporter(accessLogPath string) (*NginxExporter, error) {
 func (e *NginxExporter) Describe(ch chan<- *prometheus.Desc) {
    e.responses.Describe(ch)
    e.error_responses.Describe(ch)
-   e.bytes_sent.Describe(ch)
-   e.error_bytes_sent.Describe(ch)
 }
 
 // Implements prometheus.Collector
 func (e *NginxExporter) Collect(ch chan<- prometheus.Metric) {
    e.responses.Collect(ch)
    e.error_responses.Collect(ch)
-   e.bytes_sent.Collect(ch)
-   e.error_bytes_sent.Collect(ch)
 }
 
-var nginxProjectRe = regexp.MustCompile("(?i)^\\w+ /([^/]+)/[^\\s]* HTTP")
+var nginxProjectRe = regexp.MustCompile("(?i)^\\w+ /([^/?]+)/[^\\s]* HTTP")
 
 func (e *NginxExporter) processLogLine(line string) {
    lineReader := strings.NewReader(line)
@@ -125,9 +96,10 @@ func (e *NginxExporter) processLogLine(line string) {
    if err == nil {
       match := nginxProjectRe.FindStringSubmatch(request)
 
-      if len(match) > 1 {
+      if len(match) > 1 && IsMirroredProject(match[1]) {
          project = match[1]
       } else {
+         if len(match) > 1 { log.Println(match[1], "not found") }
          project = "none"
       }
    } else {
@@ -159,12 +131,10 @@ func (e *NginxExporter) processLogLine(line string) {
    }
 
    // Increment totals
-   e.responses.With(labels).Inc()
+   e.responses.With(labels).Observe(size)
-   e.bytes_sent.With(labels).Add(size)
 
    if !success {
-      e.error_responses.With(labels).Inc()
+      e.error_responses.With(labels).Observe(size)
-      e.error_bytes_sent.With(labels).Add(size)
    }
 }
 

proftpd.go

@@ -17,13 +17,8 @@ import (
 type ProftpdExporter struct {
    TransferLogPath string
 
-   // Request counts
+   responses *prometheus.SummaryVec
-   responses *prometheus.CounterVec
+   error_responses *prometheus.SummaryVec
-   error_responses *prometheus.CounterVec
-
-   // Bytes count
-   bytes_sent *prometheus.CounterVec
-   error_bytes_sent *prometheus.CounterVec
 }
 
 func NewProftpdExporter(transferLogPath string) (*ProftpdExporter, error) {
@@ -31,42 +26,22 @@ func NewProftpdExporter(transferLogPath string) (*ProftpdExporter, error) {
 
    return &ProftpdExporter{
       TransferLogPath: transferLogPath,
-      responses: prometheus.NewCounterVec(
+      responses: prometheus.NewSummaryVec(
-         prometheus.CounterOpts{
+         prometheus.SummaryOpts{
             Namespace: Namespace,
             Subsystem: subsystem,
-            Name: "responses_total",
+            Name: "responses",
-            Help: "Number of FTP responses",
+            Help: "Summary of FTP responses",
          },
-         []string{"project", "network", "protocol"},
+         []string{"project", "network", "protocol"}),
-      ),
+      error_responses: prometheus.NewSummaryVec(
-      error_responses: prometheus.NewCounterVec(
+         prometheus.SummaryOpts{
-         prometheus.CounterOpts{
             Namespace: Namespace,
             Subsystem: subsystem,
-            Name: "error_responses_total",
+            Name: "error_responses",
-            Help: "Number of FTP error responses",
+            Help: "Summary of error FTP responses",
          },
-         []string{"project", "network", "protocol"},
+         []string{"project", "network", "protocol"}),
-      ),
-      bytes_sent: prometheus.NewCounterVec(
-         prometheus.CounterOpts{
-            Namespace: Namespace,
-            Subsystem: subsystem,
-            Name: "responses_sent_bytes",
-            Help: "Number of bytes sent in FTP responses",
-         },
-         []string{"project", "network", "protocol"},
-      ),
-      error_bytes_sent: prometheus.NewCounterVec(
-         prometheus.CounterOpts{
-            Namespace: Namespace,
-            Subsystem: subsystem,
-            Name: "error_responses_sent_bytes",
-            Help: "Number of bytes sent in error FTP responses",
-         },
-         []string{"project", "network", "protocol"},
-      ),
    }, nil
 }
 
@@ -74,16 +49,12 @@ func NewProftpdExporter(transferLogPath string) (*ProftpdExporter, error) {
 func (e *ProftpdExporter) Describe(ch chan<- *prometheus.Desc) {
    e.responses.Describe(ch)
    e.error_responses.Describe(ch)
-   e.bytes_sent.Describe(ch)
-   e.error_bytes_sent.Describe(ch)
 }
 
 // Implements prometheus.Collector
 func (e *ProftpdExporter) Collect(ch chan<- prometheus.Metric) {
    e.responses.Collect(ch)
    e.error_responses.Collect(ch)
-   e.bytes_sent.Collect(ch)
-   e.error_bytes_sent.Collect(ch)
 }
 
 var proftpdProjectRe = regexp.MustCompile("(?i)^/mirror/root/[^/]+/([^/]+)/")
@@ -122,7 +93,7 @@ func (e *ProftpdExporter) processLogLine(line string) {
    if err == nil {
       match := proftpdProjectRe.FindStringSubmatch(request)
 
-      if len(match) > 1 {
+      if len(match) > 1 && IsMirroredProject(match[1]) {
          project = match[1]
       } else {
          project = "none"
@@ -148,12 +119,10 @@ func (e *ProftpdExporter) processLogLine(line string) {
    }
 
    // Increment totals
-   e.responses.With(labels).Inc()
+   e.responses.With(labels).Observe(size)
-   e.bytes_sent.With(labels).Add(size)
 
    if !success {
-      e.error_responses.With(labels).Inc()
+      e.error_responses.With(labels).Observe(size)
-      e.error_bytes_sent.With(labels).Add(size)
    }
 
 }

project.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+   "fmt"
+   "os"
+)
+
+func IsMirroredProject(project string) bool {
+   if _, err := os.Stat(fmt.Sprintf("/mirror/root/%s", project)); err == nil {
+      return true
+   } else {
+      return false
+   }
+}