localStorage issues

in Script development
Subscribe to localStorage issues 12 posts, 5 voices



Mango Scriptwright
FirefoxWindows

I wrote some GM_get/setValue-like functions to cache data in localStorage and automatically delete them after a specified amount of time, but when I tried to implement them in several functions in my Danbooru - Miscellaneous Tweaks script, I immediately ran into lag issues. (And I eventually noticed that FF's memory usage had ballooned on me.)

How much (key size, value size, access frequency) is too much with localStorage? (Or with GM's functions, for that matter.) And/or am I screwing up my new functions somehow? Or must there be some unrelated sort of issue causing me problems (early development bugs, interference with another script, etc...)?

On a different note, I installed Firebug for the express purpose of checking on localStorage contents, but Firebug kept displaying fewer properties than I knew were there from GM_log() output.

Only a set number (maybe 1 or 2) of expired values are removed from storage per page load (in order of size). To save space (I was capping out), I wrapped GM_xmlhttpRequest()'s onload in another onload that calls a supplied function to shrink the server response into an object with just the info I need before caching.

The major offender here is addTagSubscription(), a function that can be called multiple times (but isn't called at all by default), executes a number of queries (if another tab isn't running them), computes a raw HTML string from that, and (currently) caches that. The largest query type gave me 19k char stringified objects, with up to 40 of those for me (depends on parameters/account). The other 100+ objects were mostly simple 100 element key/int arrays. I wanted to cache each query (keying off the URL) in case of interruptions, and expire them 10 minutes later (retaining just the HTML results). Just retrieving the cached HTML (4x 1 to 8k chars, every page load) seemed to be giving me trouble. I can save more space by adding a few more queries, caching an object, and reconstructing the HTML from the object each time, but it probably isn't worth it.

Things are better now that I took them out. I'm going to try going much slower with the integration, and see what functions I can use these with...



/*
	Extra properties:
		expires: time (in s) before cached value expires
		parse: optional function (returns true on success) called before onload,
				which modifies responseDetails before saving it to cache
		sessionStorage: Use sessionStorage instead of local; expires is ignored.
*/
function MS_xmlhttpRequest(obj)
{
	var value = ( obj.sessionStorage ? JSON.parse( sessionStorage.getItem(obj.url) ) : MS_getValue(obj.url) );
	
	if( value )
		setTimeout( function(){ obj.onload(value); }, 10 );//Cached value found
	else
	{
		var oldLoad = obj.onload;		
		obj.onload = function(responseDetails)
		{
			if( responseDetails.responseText != "" && ( !obj.parse || obj.parse(responseDetails) ) )
			{
				if( !obj.sessionStorage )
					MS_setValue( obj.url, responseDetails, obj.expires || 7*24*3600 );
				else
				{
					try{ sessionStorage.setItem( obj.url, JSON.stringify(responseDetails) ); }
					catch(e) { GM_log("Clearing sessionStorage..."); for( var elem in sessionStorage ) delete sessionStorage[elem]; }
				}
			}
			oldLoad(responseDetails);
		}
		GM_xmlhttpRequest(obj);
	}
}


/**
	Store something in localStorage, to be automatically removed after a certain time.
	The value can be anything that JSON.stringify() will work on (e.g. functions will be dropped).
	
	expires: Time (in s) to wait before allowing the value to expire.  If less than 1 or not given, the value will never expire.
 */
function MS_setValue( key, value, expires )
{
	if( !key || typeof(key) != "string" )
		return;
	
	var now = Math.floor(new Date().getTime()/1000);	
	if( !expires || expires < 0 )
		expires = 0;
	else
		expires += now;
	
	key = "Misc_27776_"+key;
	value = JSON.stringify( { "value": value, "expires": expires } );
	
	//Delete old value
	MS_deleteValue(key);
	
	//Also store the key in a separate, sorted list so that expired values can be checked for later
	var keyList = JSON.parse( localStorage.getItem("Misc_27776_"+"keyList") || "[]" );
	keyList.push( { "key": key, "size": value.length, "expires": expires } );
	keyList.sort( function(a,b)
	{
		if( a <= 0 )
			return( b > 0 );//Never-expiring values go last
		if( a.expires < now && b.expires < now )
			return( b.size - a.size );//Sort expired values by decreasing size
		return( a.expires - b.expires );//Sort non-expired values by increasing date
	});
	
	try{
		localStorage.setItem( "Misc_27776_"+"keyList", JSON.stringify(keyList) );
		localStorage.setItem( key, value );
	}catch(e){
		GM_log("MS_setValue() error writing to localStorage: "+e.message);
		MS_clearCache(9999);
	}
}
function MS_getValue( key, defaultValue )
{
	key = "Misc_27776_"+key;
	var value = JSON.parse( localStorage.getItem(key) || "{}" )
	
	if( value.expires && value.expires < new Date().getTime()/1000 )
	{
		GM_log("MS_getValue(): Expired: "+key + "-----" + value.expires +" < " + (new Date().getTime()/1000)  );
		GM_deleteValue(key);
		return defaultValue;
	}
	if( value.expires == undefined )
		return defaultValue;
	
	GM_log("MS_getValue() cached result for '"+key+ (value.expires ? "' expires on "+new Date(value.expires*1000) : "' (permanent)"));
	return value.value;
}
function MS_deleteValue( key )
{
	if( !key )
		return;
	if( key.indexOf("Misc_27776_") < 0 )
		key = "Misc_27776_"+key;
	
	var keyList = JSON.parse( localStorage.getItem("Misc_27776_"+"keyList") || "[]" );
	for( var i = keyList.length - 1; i >= 0; i-- )
		if( keyList[i].key == key )
		{
			keyList.splice(i,1);
			localStorage.setItem( "Misc_27776_"+"keyList", JSON.stringify(keyList) );
			break;
		}
	localStorage.removeItem(key);
}

/**
	Removes expired values from the MS cache
	
	num: Number of values to remove.  If <= 0, entire cache is emptied regardless of expiration.
 */
function MS_clearCache(num)
{
	if( num <= 0 ) setTimeout(function(){
		for( var elem in localStorage )
			if( elem.indexOf("Misc_27776_") == 0 )
				delete localStorage[elem];
		return;
	}, 2000 );
	
	var changed = false, keyList = JSON.parse( localStorage.getItem("Misc_27776_"+"keyList") || "[]" );
	var now = new Date().getTime()/1000;
	
	while( keyList.length > 0 && keyList[0].expires < now && num-- > 0 )
	{
		changed = true;
		var elem = keyList.shift();
		GM_log("MS_clearCache(): Removed "+elem.key+" (expired on "+new Date(elem.expires*1000)+")");
		localStorage.removeItem( elem.key );
	}
	if( changed )
		localStorage.setItem( "Misc_27776_"+"keyList", JSON.stringify(keyList) );
}
MS_clearCache(1);

 
Jefferson Scher Scriptwright
FirefoxWindows

I skimmed this a couple weeks ago. Might be relevant to your project.

There is no simple solution for local storage - Mozilla Hacks – the Web developer blog

 
Mango Scriptwright
FirefoxWindows

I just can't win with this. Just calling MS_setValue() 24 times, each with an object of size 50 as a string, causes bad lag. It also seems to increase FF's memory usage with each page load.

SessionStorage doesn't cause me lag, but I'm still eying the memory usage.

 
darooooooooool Scriptwright
FirefoxWindows

One key thing i noticed, "setTimeout( function(){ obj.onload(value); }, 10 );" that 10 is 10 ms, not 10 s, at that rate you might as well not have the setTimeout. 10000 would be 10s.

 
darooooooooool Scriptwright
FirefoxWindows

Another item that may salvage you some memory by getting rid of an extra line of code:

instead of "for( var i = keyList.length - 1; i >= 0; i-- )"

try: for(var i=keyList.length;--i>=0;)

 
darooooooooool Scriptwright
FirefoxWindows

thirdly, instead of three seperate global functions. Use a list, you will have to heed scope a bit more.

example:

var myVariable = {
somechangingvariable:0,
init:function(){

"some code"
},
anotherfunction:function(){

"some more code"

}
};
myVariable.init();

This keeps all functions on a local level

 
darooooooooool Scriptwright
FirefoxWindows

I'm not going too deep into firefox hacks for performance:

Although, here are some key features you can disable safely in "about:config"

search prefetch, only one object will show up, disable it.

dom.ipc.plugins, just search for dom, disable this also.

for more tweaks, google: "about:config" firefox tweaks

 
Mango Scriptwright
FirefoxWindows

Firefox is currently taking up 680 MB of memory (it nearly reached 1 GB during my first post). I'm blaming the tentative uses of sessionStorage I replaced localStorage with in the OP code. The objects I'm storing are very small, but this still doesn't seem to work. I'll disable it all and see if that gets better. Maybe there's some completely unrelated thing that's eating my memory...

darooooooooool wrote:
One key thing i noticed, "setTimeout( function(){ obj.onload(value); }, 10 );" that 10 is 10 ms, not 10 s, at that rate you might as well not have the setTimeout. 10000 would be 10s.
That was intentional. Even such a short timeout will give other code a chance to execute instead of getting stuck in a loop until all of a chain of calls to that function are executed (if all calls are cached).

darooooooooool wrote:
Another item that may salvage you some memory by getting rid of an extra line of code:
Sacrifices clarity, gains absolutely nothing.
darooooooooool wrote:
thirdly, instead of three seperate global functions. Use a list, you will have to heed scope a bit more.
I can't see the point in this.
darooooooooool wrote:
I'm not going too deep into firefox hacks for performance:
No good. I want my script to be compatible with more than just Firefox, and I don't like trying to get everyone who uses my script to have to jump through additional hoops to avoid crazy lag/memory consumption even if there was a solution in that.

 
Couchy Scriptwright
ChromeX11

A couple points: 1.localStorage was not designed or optimized to be (ab)used by userscripts as a makeshift realtime cache. 2.Firefox is notorious for having crazy memory leaks.

All I can suggest is trying to consolidate reads/writes to localStorage as much as possible and not calling it in a loop.

 
Vivre Scriptwright
FirefoxX11

[ot]
@Mango - just like to pass on this tip that I picked up from an IT-forum to significantly reduce FF's mem-consumption:
addon Memory Fox - https://addons.mozilla.org/en-US/firefox/addon/...
[/ot]
btw - thanks for issuing this thread :-)

 
Mango Scriptwright
FirefoxWindows

Would IndexedDB be suitable for this kind of use? Employing it in a wrapper around GM_xmlhttpRequest() sounds reasonable.

I'm mostly just curious, as with the issue of cross-browser compatibility and shaky implementations, it probably isn't worth the trouble. It seemed like a fun idea, but ah well.

 
Mango Scriptwright
FirefoxWindows

I gave up on cached XMLHttpRequest and rewrote the set/get functions. The optional memObj parameter lets me stick a set of expiring values inside a single variable that calls GM_setValue on a timeout to avoid a flood of individual variables/writes. Seems to be working well. I have separate definitions for the GM functions for non-GM browsers.

function MS_setValue( key, value, expires, memObj )
{
	//If expires <= 0, expire in two weeks.
	expires = Math.floor( new Date().getTime()/1000 + (expires > 0 ? expires : 14*24*3600) );
	var keyList, remIndex = -1, addIndex = -1;
	
	if( !memObj )
	{
		GM_setValue( key, JSON.stringify({ "value": value, "expires": expires }) );
		keyList = JSON.parse( GM_getValue( "Miscellaneous.keyList", "[]" ) );
	}
	else
	{
		if( !memObj.keyList )
			memObj.keyList = [];
		keyList = memObj.keyList;
		memObj[key] = { "value": value, "expires": expires };
		
		MS_setMemObj( memObj, "MS_setValue" );
	}
	
	for( var i = 0; i < keyList.length && (addIndex < 0 || remIndex < 0 ); i++ )
	{
		if( addIndex < 0 && keyList[i].expires > expires )
			addIndex = i;
		if( remIndex < 0 && keyList[i].key == key )
			remIndex = i;
	}
	if( remIndex >= 0 )
	{
		//Key already existed; remove it.
		keyList.splice( remIndex, 1 );
		if( addIndex > remIndex )
			addIndex--;
	}
	if( addIndex < 0 )
		keyList.push({ "key": key, "expires": expires });//No other key expires later than this one.
	else
		keyList.splice( addIndex, 0, { "key": key, "expires": expires });
	
	if( !memObj )
		GM_setValue( "Miscellaneous.keyList", JSON.stringify(keyList) );
}

function MS_getValue( key, defaultValue, memObj )
{
	if( !memObj && typeof(defaultValue) == "object" && defaultValue.saveName )
	{
		memObj = defaultValue;
		defaultValue = null;
	}
	if( memObj )
	{
		if( !memObj[key] || memObj[key].expires < new Date().getTime()/1000 )
			return defaultValue;
		return memObj[key].value;
	}
	
	value = GM_getValue( key );
	
	if( !value )
		return defaultValue;
	
	try{ value = JSON.parse( value ); }
	catch(e) { return defaultValue; }
	
	if( !value.expires || value.expires < new Date().getTime()/1000 )
		return defaultValue;
	
	return value["value"];
}

function MS_deleteValue( key, memObj )
{
	var keyList = memObj ? (memObj.keyList || []) : JSON.parse( GM_getValue( "Miscellaneous.keyList", "[]" ) );
	var now = new Date().getTime()/1000;
	
	for( var i = keyList.length - 1; i >= 0 && keyList[i].expires >= 0*now; i-- )
		if( keyList[i].key == key )
		{
			keyList.splice(i,1);
			
			if( !memObj )
			{
				GM_deleteValue( key );
				GM_setValue( "Miscellaneous.keyList", JSON.stringify(keyList) );
				GM_log( "MS_deleteValue(): Deleted "+key );
			}
			else
			{
				delete memObj[key];
				MS_setMemObj( memObj, "MS_deleteValue" );
			}
			return;
		}
}

function MS_clearValues(num, memObj)
{
	if( num <= 0 )
		return;
	
	var i, keyList = (memObj ? (memObj.keyList || []) : JSON.parse( GM_getValue( "Miscellaneous.keyList", "[]" ) ) );
	var now = new Date().getTime()/1000;
	for( i = num; i > 0 && keyList.length > 0 && keyList[0].expires < now; i-- )
	{
		var key = keyList.shift().key;
		GM_deleteValue( key );
	}

	if( i != num && !memObj )
		GM_setValue( "Miscellaneous.keyList", JSON.stringify(keyList) );
} MS_clearValues(1);

function MS_setMemObj(obj,func)
{
	if( obj._timer != null )
		clearTimeout( obj._timer );
	obj._timer = setTimeout( function()
	{
		delete obj._timer;
		MS_clearValues( 20, obj );
		MS_setValue( obj.saveName, obj, obj.expires );
	}, 1500 );
}