As a follow up, here is an example of a reusable import method.
In this example you call the method like so:
doProcessFile(_oFile,doProcessLine);
This will process the read lines using the ‘doProcessLine’ function.
NOTE: make sure you don’t add the parenthesis of the function. You only want to pass the function object, not execute it.
There are 2 extra params that are optional like skipping x lines.
So just by passing the appropriate function you get the functionality you need.
I added a bunch comments in the code that explains what is required of such a function.
/**
* @param {plugins.file.JSFile} _oFile
* @param {Function} _oProcessFunction Processing function that processes each read line
* @param {Number} [_nSkipLines] Optional parameter to skip x amount of lines
* @param {Boolean} [_bUseTransaction] Optional parameter to use db transaction, default is true
*/
function doProcessFile(_oFile, _oProcessFunction, _nSkipLines, _bUseTransaction) {
// sanity check
if (_oFile === undefined || _oProcessFunction === undefined) {
// throwing an exception
throw "No file or function passed";
}
// use some default values (when needed)
_nSkipLines = (_nSkipLines !== undefined ? _nSkipLines : 0);
_bUseTransaction = (_bUseTransaction !== undefined ? _bUseTransaction : true);
//
// Use BufferedReader so we don't have to read the whole file into memory
//
var _oFR = new Packages.java.io.FileInputStream(_oFile),
_oIR = new Packages.java.io.InputStreamReader(_oFR, "UTF8"),
_oBR = new Packages.java.io.BufferedReader(_oIR),
_sLine = "dummy",
_nReadLine = 0;
if (_bUseTransaction) {
// using a database transaction (might/will) speed things up when doing inserts/updates
databaseManager.startTransaction();
}
try {
while (_sLine) {
_sLine = _oBR.readLine();
_nReadLine++;
if (_sLine && _nReadLine > _nSkipLines) {
// passing the read string, the line-counter and file object are there for reference. Your function might want to know about them.
if (!_oProcessFunction(_sLine, _nReadLine, _oFile)) {
// return true when it should continue, false when it's done
break;
}
}
}
//
//do NOT forget this close! to prevent memory leaks
//
_oBR.close();
if (_bUseTransaction) {
// Close the database transaction
databaseManager.commitTransaction();
}
} catch (_oErr) {
_oBR.close();
application.output("ERROR: " + _oFile.getName() + " at row " + _nReadLine, LOGGINGLEVEL.ERROR);
application.output("ERROR: " + _oErr, LOGGINGLEVEL.ERROR);
if (_bUseTransaction) {
databaseManager.rollbackTransaction();
}
} finally {
//
// garbage collection
//
_oFR = null;
_oIR = null;
_oBR = null;
}
}
/**
* @param {String} _sLine
* @param {Number} _nLineCount
* @param {plugins.file.JSFile} _oFile
*/
function doProcessLine(_sLine, _nLineCount, _oFile) {
// put you processing code in here
application.output(_sLine,LOGGINGLEVEL.DEBUG);
// ALWAYS RETURN TRUE OR FALSE
return true; // yes, lets keep processing.
//return false; // no, stop right now with processing.
}
This particular import does not use carriage returns/linefeeds. It is delimited by high-ascii characters.
While searching around bufferedfileReader references, I noticed that there are some character recognition capabilities with a scanner class(?) Would it be possible to use something like readLine() but which would be instead readUntilCharacter()?
The scanner class will need to load the whole file into memory to parse it so that is no good either. I guess the only way to parse it is to do it character by character.
The following function can take any single character delimiter. I tried to optimize it but it is still a factor 8-9 slower than if you use the readLine approach.
If your file uses multi-character delimiters then this will complicate things even further. But I think you have something here that gets you going in the right direction.
/**
* @param {plugins.file.JSFile} _oFile
* @param {String} _sDelimiter Delimiter character
* @param {Function} _oProcessFunction processing function that processes each read line
* @param {Number} [_nSkipLines]
* @param {Boolean} [_bUseTransaction]
*/
function doProcessFilePerChar(_oFile, _sDelimiter, _oProcessFunction, _nSkipLines, _bUseTransaction) {
// sanity check
if (_oFile === undefined || _sDelimiter === undefined || _oProcessFunction === undefined) {
// throwing an exception
throw "No file, delimiter or function passed";
}
// use some default values (when needed)
_nSkipLines = (_nSkipLines !== undefined ? _nSkipLines : 0);
_bUseTransaction = (_bUseTransaction !== undefined ? _bUseTransaction : true);
//
// Use BufferedReader so we don't have to read the whole file into memory
//
var _oFR = new Packages.java.io.FileInputStream(_oFile),
_oIR = new Packages.java.io.InputStreamReader(_oFR, "UTF8"),
_oBR = new Packages.java.io.BufferedReader(_oIR),
_nDelim = _sDelimiter.charCodeAt(0),
_aLine = new Array(),
_nChar,
_nReadLine = 0;
if (_bUseTransaction) {
// using a database transaction (might/will) speed things up
databaseManager.startTransaction();
}
try {
// read every character in the file until the end
while ( (_nChar = _oBR.read()) >= 0) {
// see if we hit the delimiter character
if (_nChar == _nDelim) {
_nReadLine++;
if (_aLine.length > 0 && _nReadLine > _nSkipLines) {
if (!_oProcessFunction(_aLine.join(""), _nReadLine, _oFile)) {
_aLine = new Array();
break;
}
}
_aLine = new Array();
} else {
// still processing the line, add it to the array
_aLine.push(String.fromCharCode(_nChar));
}
}
// we hit the end of the file, was there still some text in the array to process?
if (_aLine.length > 0) {
_nReadLine++;
if (_nReadLine > _nSkipLines) {
_oProcessFunction(_aLine.join(""), _nReadLine, _oFile)
}
}
//
//do NOT forget this close! to prevent memory leaks
//
_oBR.close();
if (_bUseTransaction) {
// Close the database transaction
databaseManager.commitTransaction();
}
} catch (_oErr) {
_oBR.close();
application.output("ERROR: " + _oFile.getName() + " at row " + _nReadLine, LOGGINGLEVEL.ERROR);
application.output("ERROR: " + _oErr, LOGGINGLEVEL.ERROR);
if (_bUseTransaction) {
databaseManager.rollbackTransaction();
}
} finally {
//
// garbage collection
//
_oFR = null;
_oIR = null;
_oBR = null;
}
}
At present I’m loading an entire 40-MB file into memory and using .indexOf() to find my field and record separations. Since the alternate approach requires reading character by character from a buffer, isn’t the .indexOf() approach going to be much faster? If this is the case, should I stay with the current approach and, if so, is there a way to clear out the memory? Although I clear the global variable (importFileText = “”) following the import, Servoy seems unable to clear its memory when I try to repeat the import.
Releasing memory back to the system once allocated to java is a tricky task and not always desirable, have a look at these java options both in the forum and in Oracle documentation:
if you do that in a webapplication and 10 users do the same (at the same time)… then only for only reading in that 40MB text file you need at least 800MB of memory, just for having those 10 files in memory…
How much memory gets used by each web client, exclusive of this import? What if there are 800 users, how much (range of) memory does that take on the server? Does it go down again when there are only 20 users, in the middle of the night?
Are there some real-world examples? (I’m working only with Developer right now.) How does Servoy deal with a 4GB, 32-bit Windows server?
Regarding the import, why 800 and not 400 MB? And why won’t it get released as soon as it is done?
it is released as soon as it is not referenced anymore, who told you other wise? I said “At the Same time”
text is UTF16 in java, so 2 bytes per character so a 40MB text file on disk in bytes is most likely 80MB in memory because of that (or the text file should itself already be UTF16 which seems unlikely, most of the time just ascii (8 byte per char) or maybe UTF8 then it is something in between 40 and 80MB)
What a webclient takes is difficult to say, depends on the solution how big the active forms are and how much data they hold on to.
4GB on a 32bit server is a waste anyway (only 3.2 or something is really there) besides that 32 bit jvm/process can only address 2GB of memory, this mostly means that your real heap space that clients use can’t be much bigger then 1.2-1.4GB because the rest is used by none heap and the vm itself.
You just have to test and see how it works for you for your specific solution and usage. Maybe it is only a few MB per active user, maybe it is 15MB per active user.
So how can I get the large file to not be referenced anymore? The problem would seem to indicate that it is still being referenced.
Can the jvm on 64-bit Windows address essentially unlimited memory?
Also, I’ve noticed that the variables on closed forms retain their values. This is helpful for some routines that refer to values on closed dialogs, but I’m wondering if I should be somehow clearing out the values to release memory. Do these forms take up heap or other memory space after they are dismissed?
it only keeps referencing to that file if you declare your variables wrong in a method
If you assign the contents of the file to a variable that you didn’t declare in a method (so it becomes a global one) then it will be kept
Or if you assign it to a form or scope variable yourself.
if you just keep it internal in a method so just the argument or just a “var tmpFile” then it is released.
Don, I think you are confusing things a bit. The headless/web client will indeed release the memory if you do as Johan suggested but this does not mean that the java VM will give the memory back to the operating system, if you check the memory usage in the servoy admin pages you won’t see the Allocated Heap Memory value going down the same applies if you use some other utility like TOP but that memory will be available for any other client running on the machine and the application server itself.
If you need your JVM to release unused memory to the system quickly have a look at those options I mentioned above in my other post.
Keep in mind though that if you are only running the servoy application server there’s no point in releasing the memory back to the system quickly and doing that could also decrease performance. Just set the minimum and maximum value and let java decide how/when to cleanup.
Regarding forms variables: forms can be destroyed (see solutionModel.removeForm()) or can be garbage collected automatically by java. You will never have exact control on how/when java will decide to garbage collect and remove forms not recently used from memory so be careful on how you use them.
Regarding forms variables: forms can be destroyed (see solutionModel.removeForm()) or can be garbage collected automatically by java. You will never have exact control on how/when java will decide to garbage collect and remove forms not recently used from memory so be careful on how you use them.
its history.remove()
solutonModel.removeForm() by itself doesn’t do much, it really doesn’t do anything if you do it with design time forms. Because design time solution model forms are never removed (just hidden if you call remove on them)
Regarding the variable which stores the file contents, it is a global variable, because of the way it is being read (function 1 → modal dialog → function 2) –
Function 1 –
function import_SELPA_central(callbackArray) {
importFileText = plugins.file.readTXTFile(callbackArray[0]);
...
var slpaCntDlgWdw = application.createWindow("slpaCntDlgWdw", JSWindow.MODAL_DIALOG); // modal dialog button calls function 2
Function 2 –
function import_SELPA_central2() {
... // process importFileText
importFileText = ""; // set importFileText to an empty string after parsing
Do I need to set importFileText = null rather than importFileText = “”, or will either of these release the memory?
So now I’m a bit confused on memory reserved for form variables –
Because design time solution model forms are never removed (just hidden if you call remove on them)
So I shouldn’t try to clear memory associated with form variables, after the form has been dismissed? For how long can I reliably obtain the variable values after the form is dismissed, by forms.formName.variableName?
now are we mixing something up. solutionModel.removeForm() removes the JSForm Solution model instance. At least if it is a solution model created form. Design time forms are never removed from memory
And remember i am talking here about the blueprint solution model forms, NOT the RuntimeForms (forms.xxxxx) those hold on to there state (like form variables). RuntimeForms are kept alive until some threshold is met (amount of memory)
then the oldest touched one is removed
If you do importFileText = “” then that is enough, that will clear the used memory. “” or null doesn’t matter.