Merge pull request #36 from DiceTechnology/improvement/ip-information

Improvement/ip information

src/main/java/technology/dice/dicewhere/api/api/IPResolver.java

@@ -207,7 +207,7 @@ public IPResolver build() throws IOException {
     }
 
     private void checkSanity() {
-      if (providers.size() <= 0) {
+      if (providers.isEmpty()) {
         throw new NoProvidersException("Must build with at least one provider");
       }
     }

src/main/java/technology/dice/dicewhere/api/api/IpInformation.java

@@ -10,6 +10,9 @@
 import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+
+import technology.dice.dicewhere.lineprocessing.serializers.protobuf.ThreeStateValueProto;
+import technology.dice.dicewhere.utils.ProtoValueConverter;
 import technology.dice.dicewhere.utils.StringUtils;
 
 public class IpInformation {
@@ -20,6 +23,7 @@ public class IpInformation {
   private final String leastSpecificDivision;
   private final String mostSpecificDivision;
   private final String postcode;
+  private final Boolean isVpn;
   private final IP startOfRange;
   private final IP endOfRange;
 
@@ -38,6 +42,7 @@ public class IpInformation {
    * @param startOfRange the first IP of the range of IPs located in this location
    * @param endOfRange the last IP of the range of IPs located in this location
    * @param originalLine the database line that got processed into this location object
+   * @param isVpn whether this range is marked as VPN from the DB provider
    */
   public IpInformation(
       @Nonnull String countryCodeAlpha2,
@@ -48,7 +53,8 @@ public IpInformation(
       @Nullable String postcode,
       @Nonnull IP startOfRange,
       @Nonnull IP endOfRange,
-      @Nullable String originalLine) {
+      @Nullable String originalLine,
+      @Nullable Boolean isVpn) {
     this.countryCodeAlpha2 = Objects.requireNonNull(countryCodeAlpha2);
     this.geonameId = geonameId;
     this.city = city;
@@ -58,6 +64,7 @@ public IpInformation(
     this.startOfRange = Objects.requireNonNull(startOfRange);
     this.endOfRange = Objects.requireNonNull(endOfRange);
     this.originalLine = originalLine;
+    this.isVpn = isVpn;
   }
 
   public String getCountryCodeAlpha2() {
@@ -96,6 +103,10 @@ public Optional<String> getOriginalLine() {
     return StringUtils.nonEmptyString(originalLine);
   }
 
+  public Optional<Boolean> isVpn() {
+    return Optional.ofNullable(isVpn);
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -113,6 +124,7 @@ public boolean equals(Object o) {
         && Objects.equals(getMostSpecificDivision(), that.getMostSpecificDivision())
         && Objects.equals(getPostcode(), that.getPostcode())
         && Objects.equals(getStartOfRange(), that.getStartOfRange())
+        && Objects.equals(isVpn(), that.isVpn())
         && Objects.equals(getEndOfRange(), that.getEndOfRange());
   }
 
@@ -127,7 +139,8 @@ public int hashCode() {
         mostSpecificDivision,
         postcode,
         startOfRange,
-        endOfRange);
+        endOfRange,
+        isVpn);
   }
 
   @Override
@@ -154,10 +167,104 @@ public String toString() {
         + ", postcode='"
         + postcode
         + '\''
+        + ", isVpn="
+        + isVpn
         + ", startOfRange="
         + startOfRange
         + ", endOfRange="
         + endOfRange
         + '}';
   }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  public static class Builder {
+    private String countryCodeAlpha2;
+    private String geonameId;
+    private String city;
+    private String leastSpecificDivision;
+    private String mostSpecificDivision;
+    private String postcode;
+    private IP startOfRange;
+    private IP endOfRange;
+    private String originalLine;
+    private Boolean isVpn;
+
+    private Builder() {}
+
+    public Builder withCountryCodeAlpha2(String countryCodeAlpha2) {
+      this.countryCodeAlpha2 = Objects.requireNonNull(countryCodeAlpha2);
+      return this;
+    }
+
+    public Builder withGeonameId(String geonameId) {
+      this.geonameId = geonameId;
+      return this;
+    }
+
+    public Builder withCity(String city) {
+      this.city = city;
+      return this;
+    }
+
+    public Builder withLeastSpecificDivision(String leastSpecificDivision) {
+      this.leastSpecificDivision = leastSpecificDivision;
+      return this;
+    }
+
+    public Builder withMostSpecificDivision(String mostSpecificDivision) {
+      this.mostSpecificDivision = mostSpecificDivision;
+      return this;
+    }
+
+    public Builder withPostcode(String postcode) {
+      this.postcode = postcode;
+      return this;
+    }
+
+    public Builder withStartOfRange(IP startOfRange) {
+      this.startOfRange = Objects.requireNonNull(startOfRange);
+      return this;
+    }
+
+    public Builder withEndOfRange(IP endOfRange) {
+      this.endOfRange = Objects.requireNonNull(endOfRange);
+      return this;
+    }
+
+    public Builder withOriginalLine(String originalLine) {
+      this.originalLine = originalLine;
+      return this;
+    }
+
+    public Builder isVpn(Optional<Boolean> isVpn) {
+      this.isVpn = isVpn.orElse(null);
+      return this;
+    }
+
+    public Builder isVpn(boolean isVpn) {
+      this.isVpn = isVpn;
+      return this;
+    }
+    public Builder isVpn(ThreeStateValueProto.ThreeStateValue isVpn) {
+      this.isVpn = ProtoValueConverter.parseThreeStateProto(isVpn).orElse(null);
+      return this;
+    }
+
+    public IpInformation build() {
+      return new IpInformation(
+          Objects.requireNonNull(countryCodeAlpha2),
+          geonameId,
+          city,
+          leastSpecificDivision,
+          mostSpecificDivision,
+          postcode,
+          Objects.requireNonNull(startOfRange),
+          Objects.requireNonNull(endOfRange),
+          originalLine,
+          isVpn);
+    }
+  }
 }

src/main/java/technology/dice/dicewhere/building/DatabaseBuilder.java

@@ -33,6 +33,7 @@ public DatabaseBuilder(
       ProviderKey provider,
       BlockingQueue<SerializedLine> source,
       DatabaseBuilderListener listener) {
+
     this.source = source;
     this.expectingMore = true;
     this.listener = listener;

src/main/java/technology/dice/dicewhere/building/IPDatabase.java

@@ -10,6 +10,7 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Optional;
+
 import technology.dice.dicewhere.api.api.IP;
 import technology.dice.dicewhere.api.api.IpInformation;
 import technology.dice.dicewhere.lineprocessing.serializers.protobuf.IPInformationProto;
@@ -32,19 +33,23 @@ public Optional<IpInformation> get(IP ip) {
                 IPInformationProto.IpInformationProto ipInformationProto =
                     IPInformationProto.IpInformationProto.parseFrom(entry.getValue());
                 IpInformation ipInformation =
-                    new IpInformation(
-                        ipInformationProto.getCountryCodeAlpha2(),
-                        ipInformationProto.getGeonameId(),
-                        ipInformationProto.getCity(),
-                        ipInformationProto.getLeastSpecificDivision(),
-                        ipInformationProto.getMostSpecificDivision(),
-                        ipInformationProto.getPostcode(),
-                        new IP(ipInformationProto.getStartOfRange().toByteArray()),
-                        new IP(ipInformationProto.getEndOfRange().toByteArray()),
-                        "".equals(ipInformationProto.getOriginalLine())
-                                || ipInformationProto.getOriginalLine() == null
-                            ? null
-                            : ipInformationProto.getOriginalLine());
+                    IpInformation.builder()
+                        .withCountryCodeAlpha2(ipInformationProto.getCountryCodeAlpha2())
+                        .withGeonameId(ipInformationProto.getGeonameId())
+                        .withCity(ipInformationProto.getCity())
+                        .withLeastSpecificDivision(ipInformationProto.getLeastSpecificDivision())
+                        .withMostSpecificDivision(ipInformationProto.getMostSpecificDivision())
+                        .withPostcode(ipInformationProto.getPostcode())
+                        .withStartOfRange(
+                            new IP(ipInformationProto.getStartOfRange().toByteArray()))
+                        .withEndOfRange(new IP(ipInformationProto.getEndOfRange().toByteArray()))
+                        .isVpn(ipInformationProto.getIsVpn())
+                        .withOriginalLine(
+                            "".equals(ipInformationProto.getOriginalLine())
+                                    || ipInformationProto.getOriginalLine() == null
+                                ? null
+                                : ipInformationProto.getOriginalLine())
+                        .build();
 
                 if (ip.isGreaterThan(ipInformation.getEndOfRange())) {
                   return Optional.empty();

src/main/java/technology/dice/dicewhere/lineprocessing/LineProcessor.java

@@ -9,22 +9,21 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Queues;
 import com.google.protobuf.ByteString;
+import technology.dice.dicewhere.api.exceptions.LineParsingException;
+import technology.dice.dicewhere.lineprocessing.serializers.protobuf.IPInformationProto.IpInformationProto;
+import technology.dice.dicewhere.parsing.LineParser;
+import technology.dice.dicewhere.parsing.ParsedLine;
+import technology.dice.dicewhere.reading.RawLine;
+import technology.dice.dicewhere.utils.ProtoValueConverter;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
 import java.util.stream.Stream;
-import technology.dice.dicewhere.api.exceptions.LineParsingException;
-import technology.dice.dicewhere.lineprocessing.serializers.protobuf.IPInformationProto.IpInformationProto;
-import technology.dice.dicewhere.parsing.LineParser;
-import technology.dice.dicewhere.parsing.ParsedLine;
-import technology.dice.dicewhere.reading.RawLine;
 
 /**
  * Responsible for processing the lines from a db provider's files and putting them in a serialized
@@ -142,9 +141,13 @@ private ImmutableList<SerializedLine> buildSerializedLineBatch(
 
   private Stream<SerializedLine> attemptParse(RawLine rawLine, long started) {
     try {
-      ParsedLine parsed = parser.parse(rawLine, retainOriginalLine);
-      progressListener.lineParsed(parsed, System.currentTimeMillis() - started);
-      return attemptSerialize(parsed);
+      Stream<ParsedLine> parsed = parser.parse(rawLine, retainOriginalLine);
+      long now = System.currentTimeMillis();
+      return parsed.flatMap(
+          l -> {
+            progressListener.lineParsed(l, now - started);
+            return attemptSerialize(l);
+          });
     } catch (LineParsingException e) {
       progressListener.parseError(rawLine, e);
       return Stream.empty();
@@ -174,7 +177,9 @@ private IpInformationProto createIpProtobuf(ParsedLine parsedLine) {
             .setMostSpecificDivision(parsedLine.getInfo().getMostSpecificDivision().orElse(""))
             .setPostcode(parsedLine.getInfo().getPostcode().orElse(""))
             .setStartOfRange(ByteString.copyFrom(parsedLine.getStartIp().getBytes()))
-            .setEndOfRange(ByteString.copyFrom(parsedLine.getEndIp().getBytes()));
+            .setEndOfRange(ByteString.copyFrom(parsedLine.getEndIp().getBytes()))
+            .setIsVpn(
+                ProtoValueConverter.toThreeStateValue(parsedLine.getInfo().isVpn().orElse(null)));
 
     parsedLine.getInfo().getOriginalLine().ifPresent(messageBuilder::setOriginalLine);
 

src/main/java/technology/dice/dicewhere/parsing/LineParser.java

@@ -9,10 +9,12 @@
 import technology.dice.dicewhere.api.exceptions.LineParsingException;
 import technology.dice.dicewhere.reading.RawLine;
 
+import java.util.stream.Stream;
+
 public interface LineParser {
-  ParsedLine parse(RawLine rawLine, boolean retainOriginalLine) throws LineParsingException;
+  Stream<ParsedLine> parse(RawLine rawLine, boolean retainOriginalLine) throws LineParsingException;
 
-  default ParsedLine parse(RawLine rawLine) throws LineParsingException {
+  default Stream<ParsedLine> parse(RawLine rawLine) throws LineParsingException {
     return parse(rawLine, false);
   }
 }

src/main/java/technology/dice/dicewhere/provider/dbip/parsing/DbIpIpToCityLiteCSVLineParser.java

@@ -19,16 +19,18 @@
 import java.net.InetAddress;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
+import java.util.stream.Stream;
 
 /**
- * Parser for DB-Ip's <a href="https://db-ip.com/db/download/ip-to-city-lite">City Lite</a> db
- * in CSV file format<br>
+ * Parser for DB-Ip's <a href="https://db-ip.com/db/download/ip-to-city-lite">City Lite</a> db in
+ * CSV file format<br>
  */
 public class DbIpIpToCityLiteCSVLineParser implements LineParser {
   private static final Splitter splitter = Splitter.on(',');
 
   @Override
-  public ParsedLine parse(RawLine line, boolean retainOriginalLine) throws LineParsingException {
+  public Stream<ParsedLine> parse(RawLine line, boolean retainOriginalLine)
+      throws LineParsingException {
     try {
       Iterable<String> fieldsIterable = splitter.split(line.getLine());
       Iterator<String> fieldsIterator = fieldsIterable.iterator();
@@ -42,22 +44,19 @@ public ParsedLine parse(RawLine line, boolean retainOriginalLine) throws LinePar
       InetAddress s = InetAddresses.forString(rangeStartString);
       IP startIp = new IP(s);
       IP endIp = new IP(e);
-      ParsedLine result =
+      return Stream.of(
           new ParsedLine(
               startIp,
               endIp,
-              new IpInformation(
-                  StringUtils.removeQuotes(countryCode),
-                  null,
-                  StringUtils.removeQuotes(city),
-                  StringUtils.removeQuotes(leastSpecificDivision),
-                  null,
-                  null,
-                  startIp,
-                  endIp,
-                  retainOriginalLine ? line.getLine() : null),
-              line);
-      return result;
+              IpInformation.builder()
+                  .withCountryCodeAlpha2(StringUtils.removeQuotes(countryCode))
+                  .withCity(StringUtils.removeQuotes(city))
+                  .withLeastSpecificDivision(StringUtils.removeQuotes(leastSpecificDivision))
+                  .withStartOfRange(startIp)
+                  .withEndOfRange(endIp)
+                  .withOriginalLine(retainOriginalLine ? line.getLine() : null)
+                  .build(),
+              line));
 
     } catch (NoSuchElementException | IllegalArgumentException e) {
       throw new LineParsingException(e, line);