Merge pull request 'пофикшено падение при получении префиксов' (#1) from hotfix/asn-fetching into master
Reviewed-on: #1
This commit is contained in:
commit
b94dbb98a4
|
|
@ -13,6 +13,7 @@ import java.net.URI;
|
|||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
|
|
@ -33,7 +34,9 @@ public class HEInfoProvider implements ASInfoProvider {
|
|||
@SneakyThrows
|
||||
public List<IPv4Subnet> getPrefixes(int as) {
|
||||
try (var client = HttpClient.newHttpClient()) {
|
||||
var request = HttpRequest.newBuilder().uri(URI.create("https://bgp.he.net/super-lg/report/api/v1/prefixes/originated/" + as)).header("Accept", "application/json").GET().build();
|
||||
var request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("https://bgp.he.net/super-lg/report/api/v1/prefixes/originated/" + as))
|
||||
.header("Accept", "application/json").timeout(Duration.ofMinutes(2)).GET().build();
|
||||
var response = client.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() == 200) {
|
||||
try (var inputStream = response.body()) {
|
||||
|
|
|
|||
|
|
@ -92,85 +92,88 @@ public class NetworkingService extends AppService {
|
|||
SystemLogger.message("Updating network manager", CTX);
|
||||
|
||||
updateProcess.set(executor.submit(() -> {
|
||||
SystemLogger.message("Update is started", CTX);
|
||||
var config = context.getConfig();
|
||||
var filteredResources = config.getFilteredResources();
|
||||
var asn = new ArrayList<>(inputResources.getASN());
|
||||
asn.removeAll(filteredResources.getASN());
|
||||
try {
|
||||
SystemLogger.message("Update is started", CTX);
|
||||
var config = context.getConfig();
|
||||
var filteredResources = config.getFilteredResources();
|
||||
var asn = new ArrayList<>(inputResources.getASN());
|
||||
asn.removeAll(filteredResources.getASN());
|
||||
|
||||
var asnToFetch = new ArrayList<>(asn);
|
||||
if (!ignoreCache) {
|
||||
asnToFetch.removeAll(prefixCache.keySet());
|
||||
}
|
||||
|
||||
fetchPrefixes(asnToFetch);
|
||||
|
||||
if (config.isCachingAS()) {
|
||||
try (var os = new FileOutputStream(cacheFile)) {
|
||||
var json = new JSONObject();
|
||||
prefixCache.forEach((key, asnList) -> {
|
||||
json.put(String.valueOf(key), JSONUtility.serializeCollection(asnList, IPv4Subnet.class, null));
|
||||
});
|
||||
os.write(json.toString().getBytes());
|
||||
} catch (IOException e) {
|
||||
SystemLogger.error("Unable to write file " + cacheFile.getPath(), CTX, e);
|
||||
var asnToFetch = new ArrayList<>(asn);
|
||||
if (!ignoreCache) {
|
||||
asnToFetch.removeAll(prefixCache.keySet());
|
||||
}
|
||||
}
|
||||
|
||||
var subnets = new HashSet<>(inputResources.getSubnets());
|
||||
asn.forEach(n -> {
|
||||
var cached = prefixCache.get(n);
|
||||
if (cached == null) {
|
||||
return;
|
||||
}
|
||||
subnets.addAll(cached);
|
||||
SystemLogger.message("Using " + cached.size() + " subnets from AS" + n, CTX);
|
||||
});
|
||||
filteredResources.getSubnets().forEach(subnets::remove);
|
||||
fetchPrefixes(asnToFetch);
|
||||
|
||||
SystemLogger.message("Trying to summary " + subnets.size() + " subnets...", CTX);
|
||||
|
||||
var merged = IPv4Util.summarySubnets(subnets, config.getMergeSubnetsWithUsage());
|
||||
var unmerged = new AtomicInteger();
|
||||
subnets.forEach(subnet -> {
|
||||
if (!merged.getMergedSubnets().contains(subnet)) {
|
||||
unmerged.getAndIncrement();
|
||||
}
|
||||
});
|
||||
|
||||
SystemLogger.message(subnets.size() + " subnets has been summarized and merged to " + merged.getResult().size() + " new subnets. Unmerged: " + unmerged.get(), CTX);
|
||||
|
||||
var domains = new HashSet<>(inputResources.getDomains());
|
||||
filteredResources.getDomains().forEach(domains::remove);
|
||||
//check domain overlaps
|
||||
|
||||
var domainsToRemove = new HashSet<String>();
|
||||
for (var domainToMatch : domains) {
|
||||
var pattern = "." + domainToMatch;
|
||||
for (var domain : domains) {
|
||||
if (domain.endsWith(pattern)) {
|
||||
domainsToRemove.add(domain);
|
||||
if (config.isCachingAS()) {
|
||||
try (var os = new FileOutputStream(cacheFile)) {
|
||||
var json = new JSONObject();
|
||||
prefixCache.forEach((key, asnList) -> {
|
||||
json.put(String.valueOf(key), JSONUtility.serializeCollection(asnList, IPv4Subnet.class, null));
|
||||
});
|
||||
os.write(json.toString().getBytes());
|
||||
} catch (IOException e) {
|
||||
SystemLogger.error("Unable to write file " + cacheFile.getPath(), CTX, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
domains.removeAll(domainsToRemove);
|
||||
var subnets = new HashSet<>(inputResources.getSubnets());
|
||||
asn.forEach(n -> {
|
||||
var cached = prefixCache.get(n);
|
||||
if (cached == null) {
|
||||
return;
|
||||
}
|
||||
subnets.addAll(cached);
|
||||
SystemLogger.message("Using " + cached.size() + " subnets from AS" + n, CTX);
|
||||
});
|
||||
filteredResources.getSubnets().forEach(subnets::remove);
|
||||
|
||||
outputResources.setASN(Collections.unmodifiableList(asn));
|
||||
outputResources.setSubnets(merged.getResult());
|
||||
outputResources.setDomains(domains.stream().toList());
|
||||
SystemLogger.message("Trying to summary " + subnets.size() + " subnets...", CTX);
|
||||
|
||||
SystemLogger.message("Update is complete", CTX);
|
||||
var merged = IPv4Util.summarySubnets(subnets, config.getMergeSubnetsWithUsage());
|
||||
var unmerged = new AtomicInteger();
|
||||
subnets.forEach(subnet -> {
|
||||
if (!merged.getMergedSubnets().contains(subnet)) {
|
||||
unmerged.getAndIncrement();
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
context.getEventsHandler().getNetworkManagerUpdateEvent().invoke(outputResources);
|
||||
SystemLogger.message(subnets.size() + " subnets has been summarized and merged to " + merged.getResult().size() + " new subnets. Unmerged: " + unmerged.get(), CTX);
|
||||
|
||||
var domains = new HashSet<>(inputResources.getDomains());
|
||||
filteredResources.getDomains().forEach(domains::remove);
|
||||
//check domain overlaps
|
||||
|
||||
var domainsToRemove = new HashSet<String>();
|
||||
for (var domainToMatch : domains) {
|
||||
var pattern = "." + domainToMatch;
|
||||
for (var domain : domains) {
|
||||
if (domain.endsWith(pattern)) {
|
||||
domainsToRemove.add(domain);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
domains.removeAll(domainsToRemove);
|
||||
|
||||
outputResources.setASN(Collections.unmodifiableList(asn));
|
||||
outputResources.setSubnets(merged.getResult());
|
||||
outputResources.setDomains(domains.stream().toList());
|
||||
|
||||
SystemLogger.message("Update is complete", CTX);
|
||||
|
||||
try {
|
||||
context.getEventsHandler().getNetworkManagerUpdateEvent().invoke(outputResources);
|
||||
} catch (Exception e) {
|
||||
SystemLogger.error("Unable to invoke update event", CTX, e);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
SystemLogger.error("Unable to invoke update event", CTX, e);
|
||||
SystemLogger.error("Something went wrong on update", CTX, e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Fetches prefixes for the given autonomous systems and stores them in the cache.
|
||||
*/
|
||||
|
|
@ -188,12 +191,11 @@ public class NetworkingService extends AppService {
|
|||
var iPv4Subnets = future.get();
|
||||
prefixCache.put(as, iPv4Subnets);
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
SystemLogger.error("Error happened while fetching AS" + as + " prefixes. Trying to use cached prefixes...", CTX, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Removes event subscriptions and shuts down the executor.
|
||||
*/
|
||||
|
|
|
|||
Loading…
Reference in New Issue