Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

If the bootstrap file is very large, the access token expires during a random batch download #7

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+40 −19
Open

If the bootstrap file is very large, the access token expires during a random batch download #7

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 40 additions & 19 deletions src/main/java/com/ebay/feed/api/FeedImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
/*
* Copyright 2018 eBay Inc.
*
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
Expand Down Expand Up @@ -50,6 +50,8 @@
import com.ebay.feed.validator.FeedValidator;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.ebay.feed.auth.CredentialLoader;
import com.ebay.feed.model.oauth.AuthRequest;

/**
* <div>
Expand All @@ -61,7 +63,7 @@
* contents</li>
* </ul>
* </div>
*
*
* @author shanganesh
*
*/
Expand All @@ -72,6 +74,7 @@ public class FeedImpl implements Feed {
private FeedUtil feedUtils = null;
private FilterUtil filterUtils = null;
private FeedValidator feedValidator = null;
static String credentialFilePath = "../../../../../../../sample-credentials/credentials.yaml";

public FeedImpl() {
client =
Expand All @@ -86,7 +89,7 @@ public FeedImpl() {

/*
* (non-Javadoc)
*
*
* @see com.ebay.feed.api.Feed#filter(java.lang.String, com.ebay.feed.model.FeedFilterRequest)
*/
@Override
Expand Down Expand Up @@ -125,7 +128,7 @@ public com.ebay.feed.model.feed.operation.filter.Response filter(FeedFilterReque

/*
* (non-Javadoc)
*
*
* @see com.ebay.feed.api.Feed#unzip(java.lang.String, boolean)
*/
@Override
Expand Down Expand Up @@ -159,7 +162,7 @@ public com.ebay.feed.model.feed.operation.filter.Response unzip(String filePath)

/*
* (non-Javadoc)
*
*
* @see com.ebay.feed.api.Feed#get(com.ebay.feed.model.FeedRequest)
*/
@Override
Expand Down Expand Up @@ -202,7 +205,7 @@ public GetFeedResponse get(FeedRequest feedRequest, String downloadDirectory) {
* additional headers including range - Creates default file/folder path and cleansup if already
* present
* </p>
*
*
* @param feedRequest
* @param downloadDirectory Optional local directory where files can be downloaded. Default is
* current working directory
Expand Down Expand Up @@ -237,7 +240,7 @@ private GetFeedResponse process(FeedRequest feedRequest, Path downloadDirectory)
// generate static request
requestBuilder = feedUtils.generateRequest(feedRequest, requestBuilder);

// generate dynamic header
// generate dynamic header
Long chunkSizeLimit = feedUtils.getChunkSizeLimit(feedRequest);
requestBuilder.addHeader(Constants.RANGE_HEADER, Constants.RANGE_PREFIX + chunkSizeLimit);

Expand All @@ -249,13 +252,13 @@ private GetFeedResponse process(FeedRequest feedRequest, Path downloadDirectory)
* <p>
* Invokes the feed API with the max range value of 100 MB. If the file is lesser than 100 MB,
* then it returns the downloaded file path along with the status.
*
*
* If the file is greater than 100 MB - Iteratively calls feed API, with incrementing range
* headers - Appends content to file - Downloads entire content and returns with downloaded file
* path.
*
*
* </p>
*
*
* @param request The API request
* @param path Path of the downloaded or partially downloading file, where contents need to be
* appended
Expand Down Expand Up @@ -295,6 +298,24 @@ private GetFeedResponse invoker(Request.Builder requestBuilder, Path path, boole

requestBuilder.removeHeader(Constants.RANGE_HEADER);
requestBuilder.addHeader(Constants.RANGE_HEADER, val);
LOGGER.debug("Firing the request for next batch" + newUpperLimit);
AuthRequest authRequest = new AuthRequest(credentialFilePath, null);

// load credentials and generate token
CredentialLoader credentialLoader = new CredentialLoader(authRequest);
String token = "foobar";
try {
credentialLoader.loadCredentials();
token = credentialLoader.getOauthResponse().getAccessToken().get().getToken();
Copy link

@danie20 danie20 May 6, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prefer to check with expire time and add logic to get token.
credentialLoader.getOauthResponse().getAccessToken().get().getExpiresOn()

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok. will make the required changes.

} catch(Exception e){
LOGGER.debug("Exception in fetching the new access token");
return new GetFeedResponse(Constants.FAILURE_CODE, Constants.FAILURE, null, null);
}
token = Constants.TOKEN_BEARER_PREFIX + token;

requestBuilder.removeHeader(Constants.AUTHORIZATION_HEADER);
requestBuilder.addHeader(Constants.AUTHORIZATION_HEADER, token);
requestBuilder.addHeader(Constants.RANGE_HEADER, val);

responseFlag = invokeIteratively(requestBuilder.build(), path);

Expand Down Expand Up @@ -325,23 +346,23 @@ private GetFeedResponse invoker(Request.Builder requestBuilder, Path path, boole
*/
private String fixFilePath(Path originalFilePath, InvokeResponse invokeResponse) {
Path newFilePath = originalFilePath;
if(originalFilePath.toString().contains("null") && !StringUtils.isEmpty(invokeResponse.getLastModified())){
if(originalFilePath.toString().contains("null") && !StringUtils.isEmpty(invokeResponse.getLastModified())){
String newPath = originalFilePath.toString().replace("null", invokeResponse.getLastModified());
newFilePath = Paths.get(newPath);
try {
Files.move(originalFilePath, newFilePath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
LOGGER.error("Unable to rename the bootstrap item feed file with date field", e);
LOGGER.error("Unable to rename the bootstrap item feed file with date field", e);
}
}
return newFilePath.toString();
}

/**
* <p>
* Invoked, only if the file size is greater than max chunk size
* </p>
*
*
* @param request The API request
* @param path Path of the downloaded or partially downloading file, where contents need to be
* appended
Expand Down Expand Up @@ -382,15 +403,15 @@ private InvokeResponse invokeIteratively(Request request, Path path) {
new InvokeResponse(response.header(Constants.CONTENT_RANGE_HEADER), response.code(), lastModifiedDate);

} catch (Throwable t) {
LOGGER.error("Exception in feed.invokeIteratively()", t);
LOGGER.error("Exception in feed.invokeIteratively()", t);
responseFlag = new InvokeResponse(null, 400);
}
return responseFlag;
}

/*
* (non-Javadoc)
*
*
* @see com.ebay.feed.api.Feed#processConfigFile(java.lang.String, java.lang.String)
*/
@Override
Expand Down Expand Up @@ -496,7 +517,7 @@ public List<com.ebay.feed.model.feed.operation.filter.Response> processConfigFil
* <p>
* Returns a new instance of response, based on the provided inputs
* </p>
*
*
* @param statusCode
* @param message
* @param filePath
Expand Down